Repository: elastic/elasticsearch
Pull request #124651: Bug Fix: System Data Streams Should Be Restorable
Author: login: JVerwolf
Labels: name: >bug
name: :Core/Infra/Core
description: Core issues without another label
name: backport pending
name: :StorageEngine/Data streams
description: Data streams and their lifecycles
name: Team:Core/Infra
description: Meta label for core/infra team
name: Team:Data Management (obsolete)
description: DO NOT USE. This team no longer exists.
name: auto-backport
description: Automatically create backport pull requests when merged
name: v8.18.1
name: v8.19.0
name: v9.0.1
name: v9.1.0
Created: 2025-03-12T14:30:22Z | Merged: 2025-03-14T15:00:44Z

This PR deletes system data streams prior to a `restore` operation.  To enable this, I refactored the way that datastreams are deleted to allow for a single metadata update operation that both deletes existing system datastreams and then restores them as part of a single operation.

closes https://github.com/elastic/elasticsearch/issues/89261


Diff:
diff --git a/docs/changelog/124651.yaml b/docs/changelog/124651.yaml
new file mode 100644
index 0000000000000..5c5e064fcd5e8
--- /dev/null
+++ b/docs/changelog/124651.yaml
@@ -0,0 +1,5 @@
+pr: 124651
+summary: "Fix system data streams to be restorable from a snapshot"
+area: Infra/Core
+type: bug
+issues: [89261]
diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DeleteDataStreamTransportAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DeleteDataStreamTransportAction.java
index 0b7743b608c68..798cb456fd42e 100644
--- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DeleteDataStreamTransportAction.java
+++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DeleteDataStreamTransportAction.java
@@ -21,9 +21,8 @@
 import org.elasticsearch.cluster.ProjectState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
-import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.metadata.MetadataDeleteIndexService;
+import org.elasticsearch.cluster.metadata.MetadataDataStreamsService;
 import org.elasticsearch.cluster.metadata.ProjectMetadata;
 import org.elasticsearch.cluster.project.ProjectResolver;
 import org.elasticsearch.cluster.service.ClusterService;
@@ -32,11 +31,8 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.SuppressForbidden;
-import org.elasticsearch.index.Index;
 import org.elasticsearch.indices.SystemIndices;
 import org.elasticsearch.injection.guice.Inject;
-import org.elasticsearch.snapshots.SnapshotInProgressException;
-import org.elasticsearch.snapshots.SnapshotsService;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
@@ -46,6 +42,7 @@
 import java.util.List;
 import java.util.Set;
 import java.util.function.Consumer;
+import java.util.stream.Collectors;
 
 import static org.elasticsearch.action.datastreams.DataStreamsActionUtil.getDataStreamNames;
 
@@ -155,34 +152,11 @@ static ClusterState removeDataStream(
             }
         }
 
-        Set<String> snapshottingDataStreams = SnapshotsService.snapshottingDataStreams(projectState, dataStreams);
-        if (snapshottingDataStreams.isEmpty() == false) {
-            throw new SnapshotInProgressException(
-                "Cannot delete data streams that are being snapshotted: "
-                    + snapshottingDataStreams
-                    + ". Try again after snapshot finishes or cancel the currently running snapshot."
-            );
-        }
-
-        Set<Index> backingIndicesToRemove = new HashSet<>();
-        for (String dataStreamName : dataStreams) {
-            DataStream dataStream = project.dataStreams().get(dataStreamName);
-            assert dataStream != null;
-            backingIndicesToRemove.addAll(dataStream.getIndices());
-            backingIndicesToRemove.addAll(dataStream.getFailureIndices());
-        }
-
-        // first delete the data streams and then the indices:
-        // (this to avoid data stream validation from failing when deleting an index that is part of a data stream
-        // without updating the data stream)
-        // TODO: change order when delete index api also updates the data stream the index to be removed is member of
-        ClusterState newState = projectState.updatedState(builder -> {
-            for (String ds : dataStreams) {
-                LOGGER.info("removing data stream [{}]", ds);
-                builder.removeDataStream(ds);
-            }
-        });
-        return MetadataDeleteIndexService.deleteIndices(newState.projectState(projectState.projectId()), backingIndicesToRemove, settings);
+        return MetadataDataStreamsService.deleteDataStreams(
+            projectState,
+            dataStreams.stream().map(project.dataStreams()::get).collect(Collectors.toSet()),
+            settings
+        );
     }
 
     @Override
diff --git a/server/build.gradle b/server/build.gradle
index 8bed775e4efbe..1afb32a973e02 100644
--- a/server/build.gradle
+++ b/server/build.gradle
@@ -80,6 +80,7 @@ dependencies {
   }
   internalClusterTestImplementation(project(':modules:reindex'))
   internalClusterTestImplementation(project(':modules:mapper-extras'))
+  internalClusterTestImplementation(project(':modules:data-streams'))
 }
 
 spotless {
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/AutoCreateSystemIndexIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/AutoCreateSystemIndexIT.java
index e46a0e2ab65ee..5def913d7f127 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/AutoCreateSystemIndexIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/AutoCreateSystemIndexIT.java
@@ -28,7 +28,7 @@
 import org.elasticsearch.indices.TestSystemIndexPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.SystemIndexPlugin;
-import org.elasticsearch.snapshots.SystemIndicesSnapshotIT;
+import org.elasticsearch.snapshots.SystemResourceSnapshotIT;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.xcontent.XContentType;
 import org.junit.After;
@@ -300,7 +300,7 @@ public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings sett
 
         @Override
         public String getFeatureName() {
-            return SystemIndicesSnapshotIT.SystemIndexTestPlugin.class.getSimpleName();
+            return SystemResourceSnapshotIT.SystemIndexTestPlugin.class.getSimpleName();
         }
 
         @Override
diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemResourceSnapshotIT.java
similarity index 62%
rename from server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java
rename to server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemResourceSnapshotIT.java
index f78c36e30aa4d..8437590278301 100644
--- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemResourceSnapshotIT.java
@@ -10,12 +10,21 @@
 package org.elasticsearch.snapshots;
 
 import org.elasticsearch.action.ActionFuture;
+import org.elasticsearch.action.DocWriteRequest;
+import org.elasticsearch.action.DocWriteResponse;
 import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
 import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse;
 import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
+import org.elasticsearch.action.datastreams.DeleteDataStreamAction;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
+import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
+import org.elasticsearch.cluster.metadata.Template;
+import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.datastreams.DataStreamsPlugin;
 import org.elasticsearch.indices.AssociatedIndexDescriptor;
+import org.elasticsearch.indices.ExecutorNames;
+import org.elasticsearch.indices.SystemDataStreamDescriptor;
 import org.elasticsearch.indices.SystemIndexDescriptor;
 import org.elasticsearch.indices.SystemIndexDescriptorUtils;
 import org.elasticsearch.plugins.Plugin;
@@ -24,10 +33,12 @@
 import org.elasticsearch.test.ESIntegTestCase;
 import org.junit.Before;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
@@ -41,9 +52,10 @@
 import static org.hamcrest.Matchers.in;
 import static org.hamcrest.Matchers.lessThan;
 import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.oneOf;
 
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0)
-public class SystemIndicesSnapshotIT extends AbstractSnapshotIntegTestCase {
+public class SystemResourceSnapshotIT extends AbstractSnapshotIntegTestCase {
 
     public static final String REPO_NAME = "test-repo";
 
@@ -55,6 +67,11 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
         plugins.add(SystemIndexTestPlugin.class);
         plugins.add(AnotherSystemIndexTestPlugin.class);
         plugins.add(AssociatedIndicesTestPlugin.class);
+        plugins.add(DataStreamsPlugin.class);
+        plugins.add(AnotherSystemDataStreamTestPlugin.class);
+        plugins.add(SystemDataStreamTestPlugin.class);
+        plugins.add(SystemDataStreamManyShardsTestPlugin.class);
+        plugins.add(AssociatedIndicesSystemDSTestPlugin.class);
         return plugins;
     }
 
@@ -70,16 +87,18 @@ public void setup() {
      */
     public void testRestoreSystemIndicesAsGlobalState() {
         createRepository(REPO_NAME, "fs");
-        // put a document in a system index
+        // put a document in a system index and data stream
         indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc");
-        refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME);
+        indexDataStream(SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME, "1", "purpose", "pre-snapshot doc");
+        refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME);
 
         // run a snapshot including global state
         createFullSnapshot(REPO_NAME, "test-snap");
 
-        // add another document
+        // add another document to each system resource
         indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "2", "purpose", "post-snapshot doc");
-        refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME);
+        indexDataStream(SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME, "2", "purpose", "post-snapshot doc");
+        refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME);
 
         assertThat(getDocCount(SystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(2L));
 
@@ -91,8 +110,9 @@ public void testRestoreSystemIndicesAsGlobalState() {
         ).setWaitForCompletion(true).setRestoreGlobalState(true).get();
         assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
 
-        // verify only the original document is restored
+        // verify only the original documents are restored
         assertThat(getDocCount(SystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(1L));
+        assertThat(getDocCount(SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME), equalTo(1L));
     }
 
     /**
@@ -101,6 +121,7 @@ public void testRestoreSystemIndicesAsGlobalState() {
     public void testSnapshotWithoutGlobalState() {
         createRepository(REPO_NAME, "fs");
         indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "system index doc");
+        indexDataStream(SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME, "1", "purpose", "pre-snapshot doc");
         indexDoc("not-a-system-index", "1", "purpose", "non system index doc");
 
         // run a snapshot without global state
@@ -122,6 +143,7 @@ public void testSnapshotWithoutGlobalState() {
 
         assertThat("not-a-system-index", in(snapshottedIndices));
         assertThat(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, not(in(snapshottedIndices)));
+        assertThat(SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME, not(in(snapshottedIndices)));
     }
 
     /**
@@ -131,23 +153,44 @@ public void testSnapshotByFeature() {
         createRepository(REPO_NAME, "fs");
         indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc");
         indexDoc(AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc");
-        refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME);
+        indexDataStream(SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME, "1", "purpose", "pre-snapshot doc");
+        indexDataStream(AnotherSystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME, "1", "purpose", "pre-snapshot doc");
+        refresh(
+            SystemIndexTestPlugin.SYSTEM_INDEX_NAME,
+            AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME,
+            SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME,
+            AnotherSystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME
+        );
 
         // snapshot by feature
         CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot(TEST_REQUEST_TIMEOUT, REPO_NAME, "test-snap")
             .setIncludeGlobalState(true)
             .setWaitForCompletion(true)
-            .setFeatureStates(SystemIndexTestPlugin.class.getSimpleName(), AnotherSystemIndexTestPlugin.class.getSimpleName())
+            .setFeatureStates(
+                SystemIndexTestPlugin.class.getSimpleName(),
+                AnotherSystemIndexTestPlugin.class.getSimpleName(),
+                SystemDataStreamTestPlugin.class.getSimpleName(),
+                AnotherSystemDataStreamTestPlugin.class.getSimpleName()
+            )
             .get();
         assertSnapshotSuccess(createSnapshotResponse);
 
         // add some other documents
         indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "2", "purpose", "post-snapshot doc");
         indexDoc(AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME, "2", "purpose", "post-snapshot doc");
-        refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME);
+        indexDataStream(SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME, "2", "purpose", "post-snapshot doc");
+        indexDataStream(AnotherSystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME, "2", "purpose", "post-snapshot doc");
+        refresh(
+            SystemIndexTestPlugin.SYSTEM_INDEX_NAME,
+            AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME,
+            SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME,
+            AnotherSystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME
+        );
 
         assertThat(getDocCount(SystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(2L));
         assertThat(getDocCount(AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(2L));
+        assertThat(getDocCount(SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME), equalTo(2L));
+        assertThat(getDocCount(AnotherSystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME), equalTo(2L));
 
         // restore indices as global state without closing the index
         RestoreSnapshotResponse restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot(
@@ -160,6 +203,8 @@ public void testSnapshotByFeature() {
         // verify only the original document is restored
         assertThat(getDocCount(SystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(1L));
         assertThat(getDocCount(SystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(1L));
+        assertThat(getDocCount(SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME), equalTo(1L));
+        assertThat(getDocCount(AnotherSystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME), equalTo(1L));
     }
 
     /**
@@ -175,7 +220,8 @@ public void testDefaultRestoreOnlyRegularIndices() {
 
         indexDoc(regularIndex, "1", "purpose", "create an index that can be restored");
         indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc");
-        refresh(regularIndex, SystemIndexTestPlugin.SYSTEM_INDEX_NAME);
+        indexDataStream(SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME, "1", "purpose", "pre-snapshot doc");
+        refresh(regularIndex, SystemIndexTestPlugin.SYSTEM_INDEX_NAME, SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME);
 
         // snapshot including global state
         CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot(TEST_REQUEST_TIMEOUT, REPO_NAME, "test-snap")
@@ -193,7 +239,11 @@ public void testDefaultRestoreOnlyRegularIndices() {
         assertThat(restoreResponse.getRestoreInfo().totalShards(), greaterThan(0));
         assertThat(
             restoreResponse.getRestoreInfo().indices(),
-            allOf(hasItem(regularIndex), not(hasItem(SystemIndexTestPlugin.SYSTEM_INDEX_NAME)))
+            allOf(
+                hasItem(regularIndex),
+                not(hasItem(SystemIndexTestPlugin.SYSTEM_INDEX_NAME)),
+                not(hasItem(SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME))
+            )
         );
     }
 
@@ -207,7 +257,15 @@ public void testRestoreByFeature() {
         indexDoc(regularIndex, "1", "purpose", "create an index that can be restored");
         indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc");
         indexDoc(AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc");
-        refresh(regularIndex, SystemIndexTestPlugin.SYSTEM_INDEX_NAME, AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME);
+        indexDataStream(SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME, "1", "purpose", "pre-snapshot doc");
+        indexDataStream(AnotherSystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME, "1", "purpose", "pre-snapshot doc");
+        refresh(
+            regularIndex,
+            SystemIndexTestPlugin.SYSTEM_INDEX_NAME,
+            AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME,
+            SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME,
+            AnotherSystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME
+        );
 
         // snapshot including global state
         CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot(TEST_REQUEST_TIMEOUT, REPO_NAME, "test-snap")
@@ -219,10 +277,19 @@ public void testRestoreByFeature() {
         // add some other documents
         indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "2", "purpose", "post-snapshot doc");
         indexDoc(AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME, "2", "purpose", "post-snapshot doc");
-        refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME);
-
+        indexDataStream(SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME, "2", "purpose", "post-snapshot doc");
+        indexDataStream(AnotherSystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME, "2", "purpose", "post-snapshot doc");
+        refresh(
+            regularIndex,
+            SystemIndexTestPlugin.SYSTEM_INDEX_NAME,
+            AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME,
+            SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME,
+            AnotherSystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME
+        );
         assertThat(getDocCount(SystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(2L));
         assertThat(getDocCount(AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(2L));
+        assertThat(getDocCount(SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME), equalTo(2L));
+        assertThat(getDocCount(AnotherSystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME), equalTo(2L));
 
         // Delete the regular index so we can restore it
         assertAcked(cluster().client().admin().indices().prepareDelete(regularIndex));
@@ -232,14 +299,16 @@ public void testRestoreByFeature() {
             TEST_REQUEST_TIMEOUT,
             REPO_NAME,
             "test-snap"
-        ).setWaitForCompletion(true).setFeatureStates("SystemIndexTestPlugin").get();
+        ).setWaitForCompletion(true).setFeatureStates("SystemIndexTestPlugin", "SystemDataStreamTestPlugin").get();
         assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
 
-        // verify that the restored system index has only one document
+        // verify that the restored system index and data stream each only have one document
         assertThat(getDocCount(SystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(1L));
+        assertThat(getDocCount(SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME), equalTo(1L));
 
-        // but the non-requested feature should still have its new document
+        // but the non-requested features should still have their new documents
         assertThat(getDocCount(AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(2L));
+        assertThat(getDocCount(AnotherSystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME), equalTo(2L));
     }
 
     /**
@@ -254,36 +323,58 @@ public void testSnapshotAndRestoreAssociatedIndices() {
         indexDoc(regularIndex, "1", "purpose", "pre-snapshot doc");
         indexDoc(AssociatedIndicesTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc");
         indexDoc(AssociatedIndicesTestPlugin.ASSOCIATED_INDEX_NAME, "1", "purpose", "pre-snapshot doc");
-        refresh(regularIndex, AssociatedIndicesTestPlugin.SYSTEM_INDEX_NAME, AssociatedIndicesTestPlugin.ASSOCIATED_INDEX_NAME);
+        indexDataStream(AssociatedIndicesSystemDSTestPlugin.SYSTEM_DATASTREAM_NAME, "1", "purpose", "pre-snapshot doc");
+        indexDoc(AssociatedIndicesSystemDSTestPlugin.ASSOCIATED_INDEX_NAME, "1", "purpose", "pre-snapshot doc");
+
+        refresh(
+            regularIndex,
+            AssociatedIndicesTestPlugin.SYSTEM_INDEX_NAME,
+            AssociatedIndicesTestPlugin.ASSOCIATED_INDEX_NAME,
+            AssociatedIndicesSystemDSTestPlugin.SYSTEM_DATASTREAM_NAME,
+            AssociatedIndicesSystemDSTestPlugin.ASSOCIATED_INDEX_NAME
+        );
 
         // snapshot
         CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot(TEST_REQUEST_TIMEOUT, REPO_NAME, "test-snap")
-            .setFeatureStates(AssociatedIndicesTestPlugin.class.getSimpleName())
+            .setFeatureStates(AssociatedIndicesTestPlugin.class.getSimpleName(), AssociatedIndicesSystemDSTestPlugin.class.getSimpleName())
             .setWaitForCompletion(true)
             .get();
         assertSnapshotSuccess(createSnapshotResponse);
 
         // verify the correctness of the snapshot
-        Set<String> snapshottedIndices = clusterAdmin().prepareGetSnapshots(TEST_REQUEST_TIMEOUT, REPO_NAME)
-            .get()
-            .getSnapshots()
+        var snapshotsResponse = clusterAdmin().prepareGetSnapshots(TEST_REQUEST_TIMEOUT, REPO_NAME).get();
+        Set<String> snapshottedIndices = snapshotsResponse.getSnapshots()
             .stream()
             .map(SnapshotInfo::indices)
             .flatMap(Collection::stream)
             .collect(Collectors.toSet());
+        Set<String> snapshottedDataStreams = snapshotsResponse.getSnapshots()
+            .stream()
+            .map(SnapshotInfo::dataStreams)
+            .flatMap(Collection::stream)
+            .collect(Collectors.toSet());
         assertThat(snapshottedIndices, hasItem(AssociatedIndicesTestPlugin.SYSTEM_INDEX_NAME));
         assertThat(snapshottedIndices, hasItem(AssociatedIndicesTestPlugin.ASSOCIATED_INDEX_NAME));
+        assertThat(snapshottedDataStreams, hasItem(AssociatedIndicesSystemDSTestPlugin.SYSTEM_DATASTREAM_NAME));
+        assertThat(snapshottedIndices, hasItem(AssociatedIndicesSystemDSTestPlugin.ASSOCIATED_INDEX_NAME));
 
         // add some other documents
         indexDoc(regularIndex, "2", "purpose", "post-snapshot doc");
         indexDoc(AssociatedIndicesTestPlugin.SYSTEM_INDEX_NAME, "2", "purpose", "post-snapshot doc");
-        refresh(regularIndex, AssociatedIndicesTestPlugin.SYSTEM_INDEX_NAME);
+        indexDataStream(AssociatedIndicesSystemDSTestPlugin.SYSTEM_DATASTREAM_NAME, "2", "purpose", "post-snapshot doc");
+        refresh(regularIndex, AssociatedIndicesTestPlugin.SYSTEM_INDEX_NAME, AssociatedIndicesSystemDSTestPlugin.SYSTEM_DATASTREAM_NAME);
 
         assertThat(getDocCount(regularIndex), equalTo(2L));
         assertThat(getDocCount(AssociatedIndicesTestPlugin.SYSTEM_INDEX_NAME), equalTo(2L));
+        assertThat(getDocCount(AssociatedIndicesSystemDSTestPlugin.SYSTEM_DATASTREAM_NAME), equalTo(2L));
 
         // And delete the associated index so we can restore it
-        assertAcked(indicesAdmin().prepareDelete(AssociatedIndicesTestPlugin.ASSOCIATED_INDEX_NAME).get());
+        assertAcked(
+            indicesAdmin().prepareDelete(
+                AssociatedIndicesTestPlugin.ASSOCIATED_INDEX_NAME,
+                AssociatedIndicesSystemDSTestPlugin.ASSOCIATED_INDEX_NAME
+            ).get()
+        );
 
         // restore the feature state and its associated index
         RestoreSnapshotResponse restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot(
@@ -291,15 +382,17 @@ public void testSnapshotAndRestoreAssociatedIndices() {
             REPO_NAME,
             "test-snap"
         )
-            .setIndices(AssociatedIndicesTestPlugin.ASSOCIATED_INDEX_NAME)
+            .setIndices(AssociatedIndicesTestPlugin.ASSOCIATED_INDEX_NAME, AssociatedIndicesSystemDSTestPlugin.ASSOCIATED_INDEX_NAME)
             .setWaitForCompletion(true)
-            .setFeatureStates(AssociatedIndicesTestPlugin.class.getSimpleName())
+            .setFeatureStates(AssociatedIndicesTestPlugin.class.getSimpleName(), AssociatedIndicesSystemDSTestPlugin.class.getSimpleName())
             .get();
         assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
 
         // verify only the original document is restored
         assertThat(getDocCount(AssociatedIndicesTestPlugin.SYSTEM_INDEX_NAME), equalTo(1L));
         assertThat(getDocCount(AssociatedIndicesTestPlugin.ASSOCIATED_INDEX_NAME), equalTo(1L));
+        assertThat(getDocCount(AssociatedIndicesSystemDSTestPlugin.SYSTEM_DATASTREAM_NAME), equalTo(1L));
+        assertThat(getDocCount(AssociatedIndicesSystemDSTestPlugin.ASSOCIATED_INDEX_NAME), equalTo(1L));
     }
 
     /**
@@ -308,7 +401,8 @@ public void testSnapshotAndRestoreAssociatedIndices() {
     public void testRestoreFeatureNotInSnapshot() {
         createRepository(REPO_NAME, "fs");
         indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc");
-        refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME);
+        indexDataStream(SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME, "1", "purpose", "pre-snapshot doc");
+        refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME);
 
         // snapshot including global state
         CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot(TEST_REQUEST_TIMEOUT, REPO_NAME, "test-snap")
@@ -322,7 +416,7 @@ public void testRestoreFeatureNotInSnapshot() {
             SnapshotRestoreException.class,
             clusterAdmin().prepareRestoreSnapshot(TEST_REQUEST_TIMEOUT, REPO_NAME, "test-snap")
                 .setWaitForCompletion(true)
-                .setFeatureStates("SystemIndexTestPlugin", fakeFeatureStateName)
+                .setFeatureStates("SystemIndexTestPlugin", "SystemDataStreamTestPlugin", fakeFeatureStateName)
         );
 
         assertThat(
@@ -438,7 +532,8 @@ public void testSystemIndicesCannotBeRenamed() {
     public void testRestoreSystemIndicesAsGlobalStateWithDefaultFeatureStateList() {
         createRepository(REPO_NAME, "fs");
         indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc");
-        refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME);
+        indexDataStream(SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME, "1", "purpose", "pre-snapshot doc");
+        refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME);
 
         // run a snapshot including global state
         CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot(TEST_REQUEST_TIMEOUT, REPO_NAME, "test-snap")
@@ -449,9 +544,11 @@ public void testRestoreSystemIndicesAsGlobalStateWithDefaultFeatureStateList() {
 
         // add another document
         indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "2", "purpose", "post-snapshot doc");
-        refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME);
+        indexDataStream(SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME, "2", "purpose", "post-snapshot doc");
+        refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME);
 
         assertThat(getDocCount(SystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(2L));
+        assertThat(getDocCount(SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME), equalTo(2L));
 
         // restore indices as global state a null list of feature states
         RestoreSnapshotResponse restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot(
@@ -463,6 +560,7 @@ public void testRestoreSystemIndicesAsGlobalStateWithDefaultFeatureStateList() {
 
         // verify that the system index is destroyed
         assertThat(getDocCount(SystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(1L));
+        assertThat(getDocCount(SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME), equalTo(1L));
     }
 
     /**
@@ -473,8 +571,9 @@ public void testRestoreSystemIndicesAsGlobalStateWithNoFeatureStates() {
         createRepository(REPO_NAME, "fs");
         String regularIndex = "my-index";
         indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "1", "purpose", "pre-snapshot doc");
+        indexDataStream(SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME, "1", "purpose", "pre-snapshot doc");
         indexDoc(regularIndex, "1", "purpose", "pre-snapshot doc");
-        refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, regularIndex);
+        refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME, regularIndex);
 
         // run a snapshot including global state
         CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot(TEST_REQUEST_TIMEOUT, REPO_NAME, "test-snap")
@@ -485,10 +584,12 @@ public void testRestoreSystemIndicesAsGlobalStateWithNoFeatureStates() {
 
         // add another document
         indexDoc(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, "2", "purpose", "post-snapshot doc");
-        refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME);
+        indexDataStream(SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME, "2", "purpose", "post-snapshot doc");
+        refresh(SystemIndexTestPlugin.SYSTEM_INDEX_NAME, SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME);
 
         assertAcked(indicesAdmin().prepareDelete(regularIndex).get());
         assertThat(getDocCount(SystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(2L));
+        assertThat(getDocCount(SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME), equalTo(2L));
 
         // restore with global state and all indices but explicitly no feature states.
         RestoreSnapshotResponse restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot(
@@ -500,6 +601,7 @@ public void testRestoreSystemIndicesAsGlobalStateWithNoFeatureStates() {
 
         // verify that the system index still has the updated document, i.e. has not been restored
         assertThat(getDocCount(SystemIndexTestPlugin.SYSTEM_INDEX_NAME), equalTo(2L));
+        assertThat(getDocCount(SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME), equalTo(2L));
         // And the regular index has been restored
         assertThat(getDocCount(regularIndex), equalTo(1L));
     }
@@ -564,6 +666,8 @@ public void testSystemIndexAliasesAreAlwaysRestored() {
         // Create a system index
         final String systemIndexName = SystemIndexTestPlugin.SYSTEM_INDEX_NAME + "-1";
         indexDoc(systemIndexName, "1", "purpose", "pre-snapshot doc");
+        // Create a system data stream
+        indexDataStream(SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME, "1", "purpose", "pre-snapshot doc");
 
         // And a regular index
         // And a regular index so we can avoid matching all indices on the restore
@@ -608,6 +712,109 @@ public void testSystemIndexAliasesAreAlwaysRestored() {
 
     }
 
+    public void testSystemDataStreamAliasesAreAlwaysRestored() {
+        createRepository(REPO_NAME, "fs");
+        // Create a system data stream
+        indexDataStream(SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME, "1", "purpose", "pre-snapshot doc");
+
+        // And a regular index so we can avoid matching all indices on the restore
+        final String regularIndex = "regular-index";
+        final String regularAlias = "regular-alias";
+        indexDoc(regularIndex, "1", "purpose", "pre-snapshot doc");
+
+        // And make sure they both have aliases
+        final String systemDataStreamAlias = SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME + "-alias";
+        assertAcked(
+            indicesAdmin().prepareAliases(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT)
+                .addAlias(regularIndex, regularAlias)
+                .addAlias(SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME, systemDataStreamAlias, true)
+                .get()
+        );
+
+        // And add a doc to ensure the alias works
+        indexDataStream(systemDataStreamAlias, "2", "purpose", "post-alias doc");
+
+        // Run a snapshot including global state
+        CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot(TEST_REQUEST_TIMEOUT, REPO_NAME, "test-snap")
+            .setWaitForCompletion(true)
+            .setIncludeGlobalState(true)
+            .get();
+        assertSnapshotSuccess(createSnapshotResponse);
+
+        // And delete the regular index and system data stream
+        assertAcked(cluster().client().admin().indices().prepareDelete(regularIndex));
+        assertAcked(
+            client().execute(
+                DeleteDataStreamAction.INSTANCE,
+                new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME)
+            ).actionGet()
+        );
+
+        // Now restore the snapshot with no aliases
+        RestoreSnapshotResponse restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot(
+            TEST_REQUEST_TIMEOUT,
+            REPO_NAME,
+            "test-snap"
+        )
+            .setFeatureStates("SystemDataStreamTestPlugin")
+            .setWaitForCompletion(true)
+            .setRestoreGlobalState(false)
+            .setIncludeAliases(false)
+            .get();
+        assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
+
+        // The regular index should exist
+        assertTrue(indexExists(regularIndex));
+        assertFalse(indexExists(regularAlias));
+
+        // And the system data stream, queried by alias, should have 2 docs
+        assertTrue(indexExists(SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME));
+        assertTrue(indexExists(systemDataStreamAlias));
+        assertThat(getDocCount(systemDataStreamAlias), equalTo(2L));
+    }
+
+    public void testDeletedDatastreamIsRestorable() {
+        createRepository(REPO_NAME, "fs");
+        // Create a system data stream
+        indexDataStream(SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME, "1", "purpose", "pre-snapshot doc");
+
+        // And a regular index so we can avoid matching all indices on the restore
+        final String regularIndex = "regular-index";
+        indexDoc(regularIndex, "1", "purpose", "pre-snapshot doc");
+
+        // Run a snapshot including global state
+        CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot(TEST_REQUEST_TIMEOUT, REPO_NAME, "test-snap")
+            .setWaitForCompletion(true)
+            .setIncludeGlobalState(true)
+            .get();
+        assertSnapshotSuccess(createSnapshotResponse);
+
+        // And delete the regular index and system data stream
+        assertAcked(cluster().client().admin().indices().prepareDelete(regularIndex));
+        assertAcked(
+            client().execute(
+                DeleteDataStreamAction.INSTANCE,
+                new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME)
+            ).actionGet()
+        );
+
+        // Now restore the snapshot with no aliases
+        RestoreSnapshotResponse restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot(
+            TEST_REQUEST_TIMEOUT,
+            REPO_NAME,
+            "test-snap"
+        )
+            .setFeatureStates("SystemDataStreamTestPlugin")
+            .setWaitForCompletion(true)
+            .setRestoreGlobalState(false)
+            .setIncludeAliases(false)
+            .get();
+
+        // And the system data stream, queried by alias, should have 2 docs
+        assertTrue(indexExists(SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME));
+        assertThat(getDocCount(SystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME), equalTo(1L));
+    }
+
     /**
      * Tests that the special "none" feature state name cannot be combined with other
      * feature state names, and an error occurs if it's tried.
@@ -691,6 +898,7 @@ public void testNoneFeatureStateOnCreation() {
         assertThat(snapshottedIndices, allOf(hasItem(regularIndex), not(hasItem(SystemIndexTestPlugin.SYSTEM_INDEX_NAME))));
     }
 
+    // TODO, Do we need to test this for Datastreams?
     /**
      * Ensures that if we can only capture a partial snapshot of a system index, then the feature state associated with that index is
      * not included in the snapshot, because it would not be safe to restore that feature state.
@@ -744,6 +952,60 @@ public void testPartialSnapshotsOfSystemIndexRemovesFeatureState() throws Except
         });
     }
 
+    /**
+     * Ensures that if we can only capture a partial snapshot of a system data stream, then the feature state associated
+     * with that data stream is not included in the snapshot, because it would not be safe to restore that feature state.
+     */
+    public void testPartialSnapshotsOfSystemDataStreamRemovesFeatureState() throws Exception {
+        final String partialIndexName = SystemDataStreamManyShardsTestPlugin.SYSTEM_DATASTREAM_NAME;
+        final String fullIndexName = AnotherSystemDataStreamTestPlugin.SYSTEM_DATASTREAM_NAME;
+
+        createRepositoryNoVerify(REPO_NAME, "mock");
+
+        // Create the index that we'll get a partial snapshot of with a bunch of shards
+        indexDataStream(partialIndexName, "1", "purpose", "pre-snapshot doc");
+        // And another one with the default
+        indexDataStream(fullIndexName, "1", "purpose", "pre-snapshot doc");
+        ensureGreen();
+
+        // Stop a random data node so we lose a shard from the partial index
+        internalCluster().stopRandomDataNode();
+        assertBusy(() -> {
+            var status = clusterAdmin().prepareHealth(TEST_REQUEST_TIMEOUT).get().getStatus();
+            assertThat(status, oneOf(ClusterHealthStatus.YELLOW, ClusterHealthStatus.RED));
+        }, 30, TimeUnit.SECONDS);
+
+        // Get ready to block
+        blockMasterFromFinalizingSnapshotOnIndexFile(REPO_NAME);
+
+        // Start a snapshot and wait for it to hit the block, then kill the master to force a failover
+        final String partialSnapName = "test-partial-snap";
+        CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot(
+            TEST_REQUEST_TIMEOUT,
+            REPO_NAME,
+            partialSnapName
+        ).setIncludeGlobalState(true).setWaitForCompletion(false).setPartial(true).get();
+        assertThat(createSnapshotResponse.status(), equalTo(RestStatus.ACCEPTED));
+        waitForBlock(internalCluster().getMasterName(), REPO_NAME);
+        internalCluster().stopCurrentMasterNode();
+
+        // Now get the snapshot and do our checks
+        assertBusy(() -> {
+            GetSnapshotsResponse snapshotsStatusResponse = clusterAdmin().prepareGetSnapshots(TEST_REQUEST_TIMEOUT, REPO_NAME)
+                .setSnapshots(partialSnapName)
+                .get();
+            SnapshotInfo snapshotInfo = snapshotsStatusResponse.getSnapshots().get(0);
+            assertNotNull(snapshotInfo);
+            assertThat(snapshotInfo.failedShards(), lessThan(snapshotInfo.totalShards()));
+            List<String> statesInSnapshot = snapshotInfo.featureStates().stream().map(SnapshotFeatureInfo::getPluginName).toList();
+            assertThat(statesInSnapshot, not(hasItem((new SystemDataStreamManyShardsTestPlugin()).getFeatureName())));
+            assertThat(statesInSnapshot, hasItem((new AnotherSystemDataStreamTestPlugin()).getFeatureName()));
+        }, 5L, TimeUnit.SECONDS);
+
+        // Cleanup to prevent unrelated shutdown failures
+        internalCluster().startDataOnlyNode();
+    }
+
     public void testParallelIndexDeleteRemovesFeatureState() throws Exception {
         final String indexToBeDeleted = SystemIndexTestPlugin.SYSTEM_INDEX_NAME;
         final String fullIndexName = AnotherSystemIndexTestPlugin.SYSTEM_INDEX_NAME;
@@ -819,6 +1081,14 @@ private long getDocCount(String indexName) {
         return indicesAdmin().prepareStats(indexName).get().getPrimaries().getDocs().getCount();
     }
 
+    private DocWriteResponse indexDataStream(String index, String id, String... source) {
+        var sourceWithTimestamp = new String[source.length + 2];
+        sourceWithTimestamp[0] = "@timestamp";
+        sourceWithTimestamp[1] = Long.toString(System.currentTimeMillis());
+        System.arraycopy(source, 0, sourceWithTimestamp, 2, source.length);
+        return prepareIndex(index).setId(id).setSource((Object[]) sourceWithTimestamp).setOpType(DocWriteRequest.OpType.CREATE).get();
+    }
+
     public static class SystemIndexTestPlugin extends Plugin implements SystemIndexPlugin {
 
         public static final String SYSTEM_INDEX_NAME = ".test-system-idx";
@@ -863,6 +1133,123 @@ public String getFeatureDescription() {
         }
     }
 
+    public static class SystemDataStreamTestPlugin extends Plugin implements SystemIndexPlugin {
+
+        public static final String SYSTEM_DATASTREAM_NAME = ".test-system-data-stream";
+
+        @Override
+        public Collection<SystemDataStreamDescriptor> getSystemDataStreamDescriptors() {
+            try {
+                CompressedXContent mappings = new CompressedXContent("{\"properties\":{\"name\":{\"type\":\"keyword\"}}}");
+                return Collections.singletonList(
+                    new SystemDataStreamDescriptor(
+                        SYSTEM_DATASTREAM_NAME,
+                        "system data stream test",
+                        SystemDataStreamDescriptor.Type.EXTERNAL,
+                        ComposableIndexTemplate.builder()
+                            .indexPatterns(List.of(SYSTEM_DATASTREAM_NAME)) // TODO is this correct?
+                            .template(new Template(Settings.EMPTY, mappings, null))
+                            .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate())
+                            .build(),
+                        Map.of(),
+                        List.of("product"),
+                        ExecutorNames.DEFAULT_SYSTEM_DATA_STREAM_THREAD_POOLS
+                    )
+                );
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+        }
+
+        @Override
+        public String getFeatureName() {
+            return SystemDataStreamTestPlugin.class.getSimpleName();
+        }
+
+        @Override
+        public String getFeatureDescription() {
+            return "A simple test plugin for data streams";
+        }
+    }
+
+    public static class SystemDataStreamManyShardsTestPlugin extends Plugin implements SystemIndexPlugin {
+
+        public static final String SYSTEM_DATASTREAM_NAME = ".test-system-data-stream-many-shards";
+
+        @Override
+        public Collection<SystemDataStreamDescriptor> getSystemDataStreamDescriptors() {
+            try {
+                CompressedXContent mappings = new CompressedXContent("{\"properties\":{\"name\":{\"type\":\"keyword\"}}}");
+                return Collections.singletonList(
+                    new SystemDataStreamDescriptor(
+                        SYSTEM_DATASTREAM_NAME,
+                        "system data stream test",
+                        SystemDataStreamDescriptor.Type.EXTERNAL,
+                        ComposableIndexTemplate.builder()
+                            .indexPatterns(List.of(SYSTEM_DATASTREAM_NAME)) // TODO is this correct?
+                            .template(new Template(indexSettings(6, 0).build(), mappings, null))
+                            .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate())
+                            .build(),
+                        Map.of(),
+                        List.of("product"),
+                        ExecutorNames.DEFAULT_SYSTEM_DATA_STREAM_THREAD_POOLS
+                    )
+                );
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+        }
+
+        @Override
+        public String getFeatureName() {
+            return SystemDataStreamTestPlugin.class.getSimpleName();
+        }
+
+        @Override
+        public String getFeatureDescription() {
+            return "A simple test plugin for data streams";
+        }
+    }
+
+    public static class AnotherSystemDataStreamTestPlugin extends Plugin implements SystemIndexPlugin {
+
+        public static final String SYSTEM_DATASTREAM_NAME = ".another-test-system-data-stream";
+
+        @Override
+        public Collection<SystemDataStreamDescriptor> getSystemDataStreamDescriptors() {
+            try {
+                CompressedXContent mappings = new CompressedXContent("{\"properties\":{\"name\":{\"type\":\"keyword\"}}}");
+                return Collections.singletonList(
+                    new SystemDataStreamDescriptor(
+                        SYSTEM_DATASTREAM_NAME,
+                        "another system data stream test",
+                        SystemDataStreamDescriptor.Type.EXTERNAL,
+                        ComposableIndexTemplate.builder()
+                            .indexPatterns(List.of(SYSTEM_DATASTREAM_NAME)) // TODO is this correct?
+                            .template(new Template(Settings.EMPTY, mappings, null))
+                            .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate())
+                            .build(),
+                        Map.of(),
+                        List.of("product"),
+                        ExecutorNames.DEFAULT_SYSTEM_DATA_STREAM_THREAD_POOLS
+                    )
+                );
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+        }
+
+        @Override
+        public String getFeatureName() {
+            return AnotherSystemDataStreamTestPlugin.class.getSimpleName();
+        }
+
+        @Override
+        public String getFeatureDescription() {
+            return "Another simple test plugin for data streams";
+        }
+    }
+
     public static class AssociatedIndicesTestPlugin extends Plugin implements SystemIndexPlugin {
 
         public static final String SYSTEM_INDEX_NAME = ".third-test-system-idx";
@@ -890,4 +1277,49 @@ public String getFeatureDescription() {
             return "Another simple test plugin";
         }
     }
+
+    public static class AssociatedIndicesSystemDSTestPlugin extends Plugin implements SystemIndexPlugin {
+
+        public static final String SYSTEM_DATASTREAM_NAME = ".test-system-data-stream-two";
+        public static final String ASSOCIATED_INDEX_NAME = ".associated-idx2";
+
+        @Override
+        public Collection<SystemDataStreamDescriptor> getSystemDataStreamDescriptors() {
+            try {
+                CompressedXContent mappings = new CompressedXContent("{\"properties\":{\"name\":{\"type\":\"keyword\"}}}");
+                return Collections.singletonList(
+                    new SystemDataStreamDescriptor(
+                        SYSTEM_DATASTREAM_NAME,
+                        "system data stream test",
+                        SystemDataStreamDescriptor.Type.EXTERNAL,
+                        ComposableIndexTemplate.builder()
+                            .indexPatterns(List.of(SYSTEM_DATASTREAM_NAME)) // TODO is this correct?
+                            .template(new Template(Settings.EMPTY, mappings, null))
+                            .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate())
+                            .build(),
+                        Map.of(),
+                        List.of("product"),
+                        ExecutorNames.DEFAULT_SYSTEM_DATA_STREAM_THREAD_POOLS
+                    )
+                );
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+        }
+
+        @Override
+        public Collection<AssociatedIndexDescriptor> getAssociatedIndexDescriptors() {
+            return Collections.singletonList(new AssociatedIndexDescriptor(ASSOCIATED_INDEX_NAME, "Associated indices"));
+        }
+
+        @Override
+        public String getFeatureName() {
+            return AssociatedIndicesSystemDSTestPlugin.class.getSimpleName();
+        }
+
+        @Override
+        public String getFeatureDescription() {
+            return "Another simple test plugin";
+        }
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java
index c30a9849198d1..f78ab7e4778de 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java
@@ -9,6 +9,7 @@
 
 package org.elasticsearch.cluster.metadata;
 
+import org.elasticsearch.ResourceNotFoundException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.datastreams.ModifyDataStreamsAction;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
@@ -31,16 +32,23 @@
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.logging.LogManager;
+import org.elasticsearch.logging.Logger;
+import org.elasticsearch.snapshots.SnapshotInProgressException;
+import org.elasticsearch.snapshots.SnapshotsService;
 
 import java.io.IOException;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 import java.util.function.Function;
+import java.util.stream.Collectors;
 
 /**
  * Handles data stream modification requests.
  */
 public class MetadataDataStreamsService {
-
+    private static final Logger LOGGER = LogManager.getLogger(MetadataDataStreamsService.class);
     private final ClusterService clusterService;
     private final IndicesService indicesService;
     private final DataStreamGlobalRetentionSettings globalRetentionSettings;
@@ -432,6 +440,52 @@ private static IndexAbstraction validateIndex(ProjectMetadata project, String in
         return index;
     }
 
+    /**
+     * Removes the given data stream and their backing indices from the Project State.
+     *
+     * @param projectState The project state
+     * @param dataStreams  The data streams to remove
+     * @param settings     The settings
+     * @return The updated Project State
+     */
+    public static ClusterState deleteDataStreams(ProjectState projectState, Set<DataStream> dataStreams, Settings settings) {
+        if (dataStreams.isEmpty()) {
+            return projectState.cluster();
+        }
+
+        Set<String> dataStreamNames = dataStreams.stream().map(DataStream::getName).collect(Collectors.toSet());
+        Set<String> snapshottingDataStreams = SnapshotsService.snapshottingDataStreams(projectState, dataStreamNames);
+        if (snapshottingDataStreams.isEmpty() == false) {
+            throw new SnapshotInProgressException(
+                "Cannot delete data streams that are being snapshotted: ["
+                    + String.join(", ", snapshottingDataStreams)
+                    + "]. Try again after snapshot finishes or cancel the currently running snapshot."
+            );
+        }
+
+        Set<Index> backingIndicesToRemove = new HashSet<>();
+        for (DataStream dataStream : dataStreams) {
+            assert dataStream != null;
+            if (projectState.metadata().dataStreams().get(dataStream.getName()) == null) {
+                throw new ResourceNotFoundException("data stream [" + dataStream.getName() + "] not found");
+            }
+            backingIndicesToRemove.addAll(dataStream.getIndices());
+            backingIndicesToRemove.addAll(dataStream.getFailureIndices());
+        }
+
+        // first delete the data streams and then the indices:
+        // (this to avoid data stream validation from failing when deleting an index that is part of a data stream
+        // without updating the data stream)
+        // TODO: change order when "delete index api" also updates the data stream the "index to be removed" is a member of
+        ClusterState newState = projectState.updatedState(builder -> {
+            for (DataStream ds : dataStreams) {
+                LOGGER.info("removing data stream [{}]", ds.getName());
+                builder.removeDataStream(ds.getName());
+            }
+        });
+        return MetadataDeleteIndexService.deleteIndices(newState.projectState(projectState.projectId()), backingIndicesToRemove, settings);
+    }
+
     /**
      * A cluster state update task that consists of the cluster state request and the listeners that need to be notified upon completion.
      */
diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java
index 52d479d2a93ab..acea889be07b3 100644
--- a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java
+++ b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java
@@ -35,6 +35,7 @@
 import org.elasticsearch.cluster.metadata.MappingMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.metadata.MetadataCreateIndexService;
+import org.elasticsearch.cluster.metadata.MetadataDataStreamsService;
 import org.elasticsearch.cluster.metadata.MetadataDeleteIndexService;
 import org.elasticsearch.cluster.metadata.MetadataIndexStateService;
 import org.elasticsearch.cluster.metadata.ProjectMetadata;
@@ -620,7 +621,7 @@ private static Tuple<Map<String, DataStream>, Map<String, DataStreamAlias>> getD
         Collection<String> featureStateDataStreams,
         boolean includeAliases
     ) {
-        Map<String, DataStream> dataStreams;
+        Map<String, DataStream> allDataStreams;
         Map<String, DataStreamAlias> dataStreamAliases;
         List<String> requestedDataStreams = filterIndices(
             snapshotInfo.dataStreams(),
@@ -628,20 +629,21 @@ private static Tuple<Map<String, DataStream>, Map<String, DataStreamAlias>> getD
             IndicesOptions.lenientExpand()
         );
         if (requestedDataStreams.isEmpty()) {
-            dataStreams = Map.of();
+            allDataStreams = Map.of();
             dataStreamAliases = Map.of();
         } else {
             if (globalMetadata == null) {
                 globalMetadata = repository.getSnapshotGlobalMetadata(snapshotId);
             }
             final Map<String, DataStream> dataStreamsInSnapshot = globalMetadata.getProject().dataStreams();
-            dataStreams = Maps.newMapWithExpectedSize(requestedDataStreams.size());
+            allDataStreams = Maps.newMapWithExpectedSize(requestedDataStreams.size());
+            Map<String, DataStream> systemDataStreams = new HashMap<>();
             for (String requestedDataStream : requestedDataStreams) {
                 final DataStream dataStreamInSnapshot = dataStreamsInSnapshot.get(requestedDataStream);
                 assert dataStreamInSnapshot != null : "DataStream [" + requestedDataStream + "] not found in snapshot";
 
                 if (dataStreamInSnapshot.isSystem() == false) {
-                    dataStreams.put(requestedDataStream, dataStreamInSnapshot);
+                    allDataStreams.put(requestedDataStream, dataStreamInSnapshot);
                 } else if (requestIndices.contains(requestedDataStream)) {
                     throw new IllegalArgumentException(
                         format(
@@ -650,7 +652,8 @@ private static Tuple<Map<String, DataStream>, Map<String, DataStreamAlias>> getD
                         )
                     );
                 } else if (featureStateDataStreams.contains(requestedDataStream)) {
-                    dataStreams.put(requestedDataStream, dataStreamInSnapshot);
+                    allDataStreams.put(requestedDataStream, dataStreamInSnapshot);
+                    systemDataStreams.put(requestedDataStream, dataStreamInSnapshot);
                 } else {
                     logger.debug(
                         "omitting system data stream [{}] from snapshot restoration because its feature state was not requested",
@@ -658,11 +661,12 @@ private static Tuple<Map<String, DataStream>, Map<String, DataStreamAlias>> getD
                     );
                 }
             }
-            if (includeAliases) {
+            if (includeAliases || systemDataStreams.isEmpty() == false) {
                 dataStreamAliases = new HashMap<>();
                 final Map<String, DataStreamAlias> dataStreamAliasesInSnapshot = globalMetadata.getProject().dataStreamAliases();
+                Map<String, DataStream> dataStreamsWithAliases = includeAliases ? allDataStreams : systemDataStreams;
                 for (DataStreamAlias alias : dataStreamAliasesInSnapshot.values()) {
-                    DataStreamAlias copy = alias.intersect(dataStreams.keySet()::contains);
+                    DataStreamAlias copy = alias.intersect(dataStreamsWithAliases.keySet()::contains);
                     if (copy.getDataStreams().isEmpty() == false) {
                         dataStreamAliases.put(alias.getName(), copy);
                     }
@@ -671,7 +675,7 @@ private static Tuple<Map<String, DataStream>, Map<String, DataStreamAlias>> getD
                 dataStreamAliases = Map.of();
             }
         }
-        return new Tuple<>(dataStreams, dataStreamAliases);
+        return new Tuple<>(allDataStreams, dataStreamAliases);
     }
 
     private Map<String, List<String>> getFeatureStatesToRestore(
@@ -765,6 +769,29 @@ private Set<Index> resolveSystemIndicesToDelete(ClusterState currentState, Set<S
             .collect(Collectors.toUnmodifiableSet());
     }
 
+    /**
+     * Resolves a set of datastream names that currently exist in the cluster that are part of a feature state which is
+     * about to be restored, and should therefore be removed prior to restoring those feature states from the snapshot.
+     *
+     * @param currentState           The current cluster state
+     * @param featureStatesToRestore A set of feature state names that are about to be restored
+     * @return A set of datastream names that should be removed based on the feature states being restored
+     */
+    private Set<DataStream> resolveSystemDataStreamsToDelete(ClusterState currentState, Set<String> featureStatesToRestore) {
+        if (featureStatesToRestore == null) {
+            return Collections.emptySet();
+        }
+
+        return featureStatesToRestore.stream()
+            .map(systemIndices::getFeature)
+            .filter(Objects::nonNull) // Features that aren't present on this node will be warned about in `getFeatureStatesToRestore`
+            .flatMap(feature -> feature.getDataStreamDescriptors().stream())
+            .map(SystemDataStreamDescriptor::getDataStreamName)
+            .filter(datastreamName -> currentState.metadata().getProject().dataStreams().containsKey(datastreamName))
+            .map(dataStreamName -> currentState.metadata().getProject().dataStreams().get(dataStreamName))
+            .collect(Collectors.toUnmodifiableSet());
+    }
+
     // visible for testing
     static DataStream updateDataStream(DataStream dataStream, Metadata.Builder metadata, RestoreSnapshotRequest request) {
         String dataStreamName = dataStream.getName();
@@ -1336,6 +1363,13 @@ public ClusterState execute(ClusterState currentState) {
                 settings
             );
 
+            // Clear out all existing system data streams
+            currentState = MetadataDataStreamsService.deleteDataStreams(
+                currentState.projectState(),
+                resolveSystemDataStreamsToDelete(currentState, featureStatesToRestore),
+                settings
+            );
+
             // List of searchable snapshots indices to restore
             final Set<Index> searchableSnapshotsIndices = new HashSet<>();
 
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsServiceTests.java
index 748aae1ea17c4..2ac5938fccf42 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsServiceTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsServiceTests.java
@@ -9,7 +9,11 @@
 
 package org.elasticsearch.cluster.metadata;
 
+import org.elasticsearch.ResourceNotFoundException;
+import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ProjectState;
+import org.elasticsearch.cluster.SnapshotsInProgress;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.ClusterSettings;
@@ -20,11 +24,19 @@
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.MapperServiceTestCase;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.snapshots.Snapshot;
+import org.elasticsearch.snapshots.SnapshotId;
+import org.elasticsearch.snapshots.SnapshotInProgressException;
+import org.elasticsearch.snapshots.SnapshotInfoTestUtils;
+import org.elasticsearch.test.index.IndexVersionUtils;
 
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
 
 import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.generateMapping;
 import static org.hamcrest.Matchers.containsInAnyOrder;
@@ -466,6 +478,61 @@ public void testUpdateDataStreamOptions() {
         assertThat(updatedDataStream.getDataStreamOptions(), equalTo(DataStreamOptions.EMPTY));
     }
 
+    public void testDeleteMissing() {
+        DataStream dataStream = DataStreamTestHelper.randomInstance();
+        final var projectId = randomProjectIdOrDefault();
+        ProjectState state = ClusterState.builder(ClusterName.DEFAULT)
+            .putProjectMetadata(ProjectMetadata.builder(projectId))
+            .build()
+            .projectState(projectId);
+
+        ResourceNotFoundException e = expectThrows(
+            ResourceNotFoundException.class,
+            () -> MetadataDataStreamsService.deleteDataStreams(state, Set.of(dataStream), Settings.EMPTY)
+        );
+        assertThat(e.getMessage(), containsString(dataStream.getName()));
+    }
+
+    public void testDeleteSnapshotting() {
+        String dataStreamName = randomAlphaOfLength(5);
+        Snapshot snapshot = new Snapshot("doesn't matter", new SnapshotId("snapshot name", "snapshot uuid"));
+        SnapshotsInProgress snaps = SnapshotsInProgress.EMPTY.withAddedEntry(
+            SnapshotsInProgress.Entry.snapshot(
+                snapshot,
+                true,
+                false,
+                SnapshotsInProgress.State.INIT,
+                Collections.emptyMap(),
+                List.of(dataStreamName),
+                Collections.emptyList(),
+                System.currentTimeMillis(),
+                (long) randomIntBetween(0, 1000),
+                Map.of(),
+                null,
+                SnapshotInfoTestUtils.randomUserMetadata(),
+                IndexVersionUtils.randomVersion()
+            )
+        );
+        final DataStream dataStream = DataStreamTestHelper.randomInstance(dataStreamName);
+        var projectId = randomProjectIdOrDefault();
+        ProjectState state = ClusterState.builder(ClusterName.DEFAULT)
+            .putCustom(SnapshotsInProgress.TYPE, snaps)
+            .putProjectMetadata(ProjectMetadata.builder(projectId).put(dataStream))
+            .build()
+            .projectState(projectId);
+        Exception e = expectThrows(
+            SnapshotInProgressException.class,
+            () -> MetadataDataStreamsService.deleteDataStreams(state, Set.of(dataStream), Settings.EMPTY)
+        );
+        assertEquals(
+            "Cannot delete data streams that are being snapshotted: ["
+                + dataStreamName
+                + "]. Try again after snapshot finishes "
+                + "or cancel the currently running snapshot.",
+            e.getMessage()
+        );
+    }
+
     private MapperService getMapperService(IndexMetadata im) {
         try {
             String mapping = im.mapping().source().toString();


Reviews:
user: login: JVerwolf
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-12T17:36:58Z
user: login: rjernst
body: A few minor initial comments. I suggest someone from data management review.
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-12T19:02:15Z
user: login: JVerwolf
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-12T20:15:15Z
user: login: dakrone
body: I left a couple of comments from the Data Management side, but nothing major. I agree with Ryan about trying to combine with `MetadataDataStreamsService` instead of adding a new service if possible.
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-12T20:23:17Z
user: login: alexey-ivanov-es
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-12T20:51:41Z
user: login: JVerwolf
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-12T22:52:22Z
user: login: nielsbauman
body: Thanks for working on this, John! It's looking good to me, I just left a few comments, mostly related to MP changes.
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-13T10:12:43Z
user: login: JVerwolf
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-13T14:34:10Z
user: login: JVerwolf
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-13T15:46:59Z
user: login: JVerwolf
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-13T15:54:15Z
user: login: nielsbauman
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-13T17:51:24Z
user: login: nielsbauman
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-13T17:52:23Z
user: login: nielsbauman
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-13T17:59:00Z
user: login: JVerwolf
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-13T18:37:03Z
user: login: JVerwolf
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-13T18:38:36Z
user: login: JVerwolf
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-13T19:04:34Z
user: login: JVerwolf
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-13T19:09:03Z
user: login: nielsbauman
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-13T19:11:37Z
user: login: nielsbauman
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-13T19:13:49Z
user: login: dakrone
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-13T20:57:40Z
user: login: JVerwolf
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-13T22:29:57Z
user: login: dakrone
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-13T22:45:27Z
user: login: JVerwolf
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-13T22:58:23Z
user: login: nielsbauman
body: LGTM, thanks a lot for the iterations, @JVerwolf!
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-14T13:56:22Z

Review comments:
diff_hunk: @@ -608,6 +711,69 @@ public void testSystemIndexAliasesAreAlwaysRestored() {
 
     }
 
+    // TODO: This test is failing because the assertion `assertTrue(indexExists(systemDataStreamAlias));` trips.
+    @AwaitsFix(bugUrl = "assertTrue(indexExists(systemDataStreamAlias)); is broken")
path: server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java
user: login: JVerwolf
body: This test is failing because the alias isn't restored. I'm supposing this is a bug, and not expected behaviour for system datastreams?
created_at: 2025-03-12T17:36:58Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,5 @@
+pr: 124651
+summary: "Bug Fix: System Data Streams Should Be Restorable"
path: docs/changelog/124651.yaml
user: login: rjernst
body: nit: the "Bug fix" prefix is unnecessary (it's already of type "bug"). applies to the PR title too
created_at: 2025-03-12T18:03:04Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,83 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.cluster.metadata;
+
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ProjectState;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.FixForMultiProject;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.logging.LogManager;
+import org.elasticsearch.logging.Logger;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+public class MetadataDeleteDataStreamService {
path: server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDeleteDataStreamService.java
user: login: rjernst
body: Could these methods be on the existing MetadataDataStreamService?
created_at: 2025-03-12T19:01:17Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,5 @@
+pr: 124651
+summary: "System Data Streams Should Be Restorable"
path: docs/changelog/124651.yaml
user: login: dakrone
body: Since these go into changelogs, this should be an active sentence. Something like "Fix system data streams to be restorable from a snapshot"
created_at: 2025-03-12T20:06:39Z
author_association: MEMBER
diff_hunk: @@ -510,6 +610,7 @@ public void testRestoreSystemIndicesAsGlobalStateWithNoFeatureStates() {
      *
      * However, other feature states should be unaffected.
      */
+    // TODO: determine if we can create a datastream test for this
path: server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java
user: login: JVerwolf
body: @alexey-ivanov-es , do you know if a similar test can be written for data streams? e.g. do data streams follow a similar pattern with matching descriptors?  (I can also dig into this, but wondering if you know offhand since you've recently worked in this area)
created_at: 2025-03-12T20:15:15Z
author_association: CONTRIBUTOR
diff_hunk: @@ -608,6 +711,69 @@ public void testSystemIndexAliasesAreAlwaysRestored() {
 
     }
 
+    // TODO: This test is failing because the assertion `assertTrue(indexExists(systemDataStreamAlias));` trips.
+    @AwaitsFix(bugUrl = "assertTrue(indexExists(systemDataStreamAlias)); is broken")
path: server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java
user: login: dakrone
body: Data stream aliases are similar to index aliases, where they are metadata on the data stream itself. I would expect restoring a data stream (system or otherwise) to also restore the alias for that data stream.

I tested that with a non-system data stream, the alias is restored when the data stream is restored, so I would treat this as a bug.
created_at: 2025-03-12T20:16:31Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,83 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.cluster.metadata;
+
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ProjectState;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.FixForMultiProject;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.logging.LogManager;
+import org.elasticsearch.logging.Logger;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+public class MetadataDeleteDataStreamService {
+
+    private static final Logger LOGGER = LogManager.getLogger(MetadataDeleteDataStreamService.class);
+
+    /**
+     * Removes the given data streams from the Cluster State.
path: server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDeleteDataStreamService.java
user: login: dakrone
body: ```suggestion
     * Removes the given data streams and their backing indices from the Cluster State.
```
created_at: 2025-03-12T20:21:45Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,83 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.cluster.metadata;
+
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ProjectState;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.FixForMultiProject;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.logging.LogManager;
+import org.elasticsearch.logging.Logger;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+public class MetadataDeleteDataStreamService {
+
+    private static final Logger LOGGER = LogManager.getLogger(MetadataDeleteDataStreamService.class);
+
+    /**
+     * Removes the given data streams from the Cluster State.
+     *
+     * @param clusterState The cluster state
+     * @param dataStreams  The data streams to remove
+     * @param settings     The settings
+     * @return The updated Cluster State
+     */
+    @FixForMultiProject // Once callers have Project State we can update/remove this method.
+    public static ClusterState deleteDataStreams(ClusterState clusterState, Set<DataStream> dataStreams, Settings settings) {
+        final Map<ProjectId, Set<DataStream>> byProject = new HashMap<>();
+        for (DataStream dataStream : dataStreams) {
+            final ProjectMetadata project = clusterState.metadata().projectFor(dataStream);
+            byProject.computeIfAbsent(project.id(), ignore -> new HashSet<>()).add(dataStream);
+        }
+
+        for (final Map.Entry<ProjectId, Set<DataStream>> entry : byProject.entrySet()) {
+            clusterState = deleteDataStream(clusterState.projectState(entry.getKey()), entry.getValue(), settings);
+        }
+        return clusterState;
+    }
+
+    /**
+     * Removes the given data streams from the Project State.
path: server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDeleteDataStreamService.java
user: login: dakrone
body: ```suggestion
     * Removes the given data streams and their backing indices from the Project State.
```
created_at: 2025-03-12T20:21:58Z
author_association: MEMBER
diff_hunk: @@ -510,6 +610,7 @@ public void testRestoreSystemIndicesAsGlobalStateWithNoFeatureStates() {
      *
      * However, other feature states should be unaffected.
      */
+    // TODO: determine if we can create a datastream test for this
path: server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java
user: login: alexey-ivanov-es
body: System data streams themselves don't have a name pattern - just a name, so it is not possible to have another data stream in the same pattern
created_at: 2025-03-12T20:51:41Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,5 @@
+pr: 124651
+summary: "System Data Streams Should Be Restorable"
path: docs/changelog/124651.yaml
user: login: JVerwolf
body: Good point, thanks!
created_at: 2025-03-12T22:52:22Z
author_association: CONTRIBUTOR
diff_hunk: @@ -164,25 +163,11 @@ static ClusterState removeDataStream(
             );
path: modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DeleteDataStreamTransportAction.java
user: login: nielsbauman
body: Github doesn't allow me to target the lines above, but do we need to do the `snapshottingDataStreams` check twice (i.e. here and in `MetadataDataStreamsService#deleteDataStream`)?
created_at: 2025-03-13T08:16:09Z
author_association: CONTRIBUTOR
diff_hunk: @@ -432,6 +442,71 @@ private static IndexAbstraction validateIndex(ProjectMetadata project, String in
         return index;
     }
 
+    /**
+     * Removes the given data streams and their backing indices from the Cluster State.
+     *
+     * @param clusterState The cluster state
+     * @param dataStreams  The data streams to remove
+     * @param settings     The settings
+     * @return The updated Cluster State
+     */
+    @FixForMultiProject // Once callers have Project State we can update/remove this method.
+    public static ClusterState deleteDataStreams(ClusterState clusterState, Set<DataStream> dataStreams, Settings settings) {
path: server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java
user: login: nielsbauman
body: I prefer that we remove this overloaded method and just stick to the one that accepts a `ProjectState`, then we also don't need the two lookup methods for data streams in `Metadata.java`. AFAICS, the only usage of this version is in `RestoreService`. Just like you're using `Metadata#getProject()` there already, you can use `Metadata#projectState()` to pass a `ProjectState` of the default project to this method. Sorry if that wasn't clear from my answer in the multi-project channel.
created_at: 2025-03-13T08:22:09Z
author_association: CONTRIBUTOR
diff_hunk: @@ -432,6 +442,71 @@ private static IndexAbstraction validateIndex(ProjectMetadata project, String in
         return index;
     }
 
+    /**
+     * Removes the given data streams and their backing indices from the Cluster State.
+     *
+     * @param clusterState The cluster state
+     * @param dataStreams  The data streams to remove
+     * @param settings     The settings
+     * @return The updated Cluster State
+     */
+    @FixForMultiProject // Once callers have Project State we can update/remove this method.
+    public static ClusterState deleteDataStreams(ClusterState clusterState, Set<DataStream> dataStreams, Settings settings) {
+        final Map<ProjectId, Set<DataStream>> byProject = new HashMap<>();
+        for (DataStream dataStream : dataStreams) {
+            final ProjectMetadata project = clusterState.metadata().projectFor(dataStream);
+            byProject.computeIfAbsent(project.id(), ignore -> new HashSet<>()).add(dataStream);
+        }
+
+        for (final Map.Entry<ProjectId, Set<DataStream>> entry : byProject.entrySet()) {
+            clusterState = deleteDataStream(clusterState.projectState(entry.getKey()), entry.getValue(), settings);
+        }
+        return clusterState;
+    }
+
+    /**
+     * Removes the given data stream and their backing indices from the Project State.
+     *
+     * @param projectState The project state
+     * @param dataStreams  The data streams to remove
+     * @param settings     The settings
+     * @return The updated Project State
+     */
+    public static ClusterState deleteDataStream(ProjectState projectState, Set<DataStream> dataStreams, Settings settings) {
+        if (dataStreams.isEmpty()) {
+            return projectState.cluster();
+        }
+
+        Set<String> dataStreamNames = dataStreams.stream().map(DataStream::getName).collect(Collectors.toSet());
+        Set<String> snapshottingDataStreams = SnapshotsService.snapshottingDataStreams(projectState, dataStreamNames);
+        if (snapshottingDataStreams.isEmpty() == false) {
+            throw new SnapshotInProgressException(
+                "Cannot delete data streams that are being snapshotted: ["
+                    + String.join(", ", snapshottingDataStreams)
+                    + "]. Try again after snapshot finishes or cancel the currently running snapshot."
+            );
+        }
+
+        Set<Index> backingIndicesToRemove = new HashSet<>();
+        for (DataStream dataStream : dataStreams) {
+            assert dataStream != null;
+            backingIndicesToRemove.addAll(dataStream.getIndices());
+            backingIndicesToRemove.addAll(dataStream.getFailureIndices());
+        }
+
+        // first delete the data streams and then the indices:
+        // (this to avoid data stream validation from failing when deleting an index that is part of a data stream
+        // without updating the data stream)
+        // TODO: change order when "delete index api" also updates the data stream the "index to be removed" is a member of
+        ClusterState newState = projectState.updatedState(builder -> {
+            dataStreams.stream().map(DataStream::getName).forEach(ds -> {
+                LOGGER.info("removing data stream [{}]", ds);
+                builder.removeDataStream(ds);
+            });
path: server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java
user: login: nielsbauman
body: Nit, I don't think we need to use streams here.
```suggestion
            for (String ds : dataStreams) {
                LOGGER.info("removing data stream [{}]", ds.getName());
                builder.removeDataStream(ds.getName());
            });
```
created_at: 2025-03-13T09:40:09Z
author_association: CONTRIBUTOR
diff_hunk: @@ -691,6 +898,7 @@ public void testNoneFeatureStateOnCreation() {
         assertThat(snapshottedIndices, allOf(hasItem(regularIndex), not(hasItem(SystemIndexTestPlugin.SYSTEM_INDEX_NAME))));
     }
 
+    // TODO, Do we need to test this for Datastreams?
path: server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemResourceSnapshotIT.java
user: login: nielsbauman
body: Just commenting on this line so we don't forget this TODO. I don't know enough about (partial) snapshots to know if this is something we need to cover. @dakrone, any thoughts?
created_at: 2025-03-13T09:53:24Z
author_association: CONTRIBUTOR
diff_hunk: @@ -466,6 +478,51 @@ public void testUpdateDataStreamOptions() {
         assertThat(updatedDataStream.getDataStreamOptions(), equalTo(DataStreamOptions.EMPTY));
     }
 
+    public void testDeleteMissing() {
+        DataStream dataStream = DataStreamTestHelper.randomInstance();
+        ClusterState state = ClusterState.builder(ClusterName.DEFAULT).build();
+        IllegalArgumentException e = expectThrows(
+            IllegalArgumentException.class,
+            () -> MetadataDataStreamsService.deleteDataStreams(state, Set.of(dataStream), Settings.EMPTY)
+        );
+        assertThat(e.getMessage(), containsString(dataStream.getName()));
+    }
+
+    public void testDeleteSnapshotting() {
+        String dataStreamName = randomAlphaOfLength(5);
+        Snapshot snapshot = new Snapshot("doesn't matter", new SnapshotId("snapshot name", "snapshot uuid"));
+        SnapshotsInProgress snaps = SnapshotsInProgress.EMPTY.withAddedEntry(
+            SnapshotsInProgress.Entry.snapshot(
+                snapshot,
+                true,
+                false,
+                SnapshotsInProgress.State.INIT,
+                Collections.emptyMap(),
+                List.of(dataStreamName),
+                Collections.emptyList(),
+                System.currentTimeMillis(),
+                (long) randomIntBetween(0, 1000),
+                Map.of(),
+                null,
+                SnapshotInfoTestUtils.randomUserMetadata(),
+                IndexVersionUtils.randomVersion()
+            )
+        );
+        final DataStream dataStream = DataStreamTestHelper.randomInstance(dataStreamName);
+        ClusterState state = ClusterState.builder(clusterState(dataStream)).putCustom(SnapshotsInProgress.TYPE, snaps).build();
path: server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsServiceTests.java
user: login: nielsbauman
body: ```suggestion
		final var projectId = randomProjectIdOrDefault();
        final ProjectState state = ClusterState.builder(ClusterName.DEFAULT)
            .putCustom(SnapshotsInProgress.TYPE, snaps).build();
            .putProjectMetadata(ProjectMetadata.builder(project).put(dataStream))
            .routingTable(GlobalRoutingTableTestHelper.buildRoutingTable(metadata, RoutingTable.Builder::addAsNew))
            .build()
            .projectState(projectId);
```

Since we're removing the overloaded method, we should pass a `ProjectState` here as well. That way, we'll also fix the failing test. Additionally, since we're removing the method that loops over the projects, we don't need to test with multiple projects. That in turn removes the need of the `clusterState` method below as this was its only usage.
created_at: 2025-03-13T10:05:58Z
author_association: CONTRIBUTOR
path: server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsServiceTests.java
user: login: nielsbauman
body: I think we should move some of the tests in [DeleteDataStreamTransportActionTests.java](https://github.com/elastic/elasticsearch/blob/9cbb26247bad7ee50b3dcbea4ee64ed5deb14409/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/DeleteDataStreamTransportActionTests.java#L40-L40) to this class now that we're moving the functionality.
created_at: 2025-03-13T10:10:36Z
author_association: CONTRIBUTOR
diff_hunk: @@ -432,6 +442,71 @@ private static IndexAbstraction validateIndex(ProjectMetadata project, String in
         return index;
     }
 
+    /**
+     * Removes the given data streams and their backing indices from the Cluster State.
+     *
+     * @param clusterState The cluster state
+     * @param dataStreams  The data streams to remove
+     * @param settings     The settings
+     * @return The updated Cluster State
+     */
+    @FixForMultiProject // Once callers have Project State we can update/remove this method.
+    public static ClusterState deleteDataStreams(ClusterState clusterState, Set<DataStream> dataStreams, Settings settings) {
path: server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java
user: login: JVerwolf
body: Thanks @nielsbauman .  Since I'm still developing familiarity with the new multiproject code, is this the way I should be getting the project state at the call site? : 
```
currentState.projectState(currentState.metadata().getProject().id())
``` 
It seems convoluted, but I don't see another way to do it...
created_at: 2025-03-13T14:34:09Z
author_association: CONTRIBUTOR
diff_hunk: @@ -164,25 +163,11 @@ static ClusterState removeDataStream(
             );
path: modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DeleteDataStreamTransportAction.java
user: login: JVerwolf
body: Nice catch!
created_at: 2025-03-13T15:46:59Z
author_association: CONTRIBUTOR
diff_hunk: @@ -466,6 +478,51 @@ public void testUpdateDataStreamOptions() {
         assertThat(updatedDataStream.getDataStreamOptions(), equalTo(DataStreamOptions.EMPTY));
     }
 
+    public void testDeleteMissing() {
+        DataStream dataStream = DataStreamTestHelper.randomInstance();
+        ClusterState state = ClusterState.builder(ClusterName.DEFAULT).build();
+        IllegalArgumentException e = expectThrows(
+            IllegalArgumentException.class,
+            () -> MetadataDataStreamsService.deleteDataStreams(state, Set.of(dataStream), Settings.EMPTY)
+        );
+        assertThat(e.getMessage(), containsString(dataStream.getName()));
+    }
+
+    public void testDeleteSnapshotting() {
+        String dataStreamName = randomAlphaOfLength(5);
+        Snapshot snapshot = new Snapshot("doesn't matter", new SnapshotId("snapshot name", "snapshot uuid"));
+        SnapshotsInProgress snaps = SnapshotsInProgress.EMPTY.withAddedEntry(
+            SnapshotsInProgress.Entry.snapshot(
+                snapshot,
+                true,
+                false,
+                SnapshotsInProgress.State.INIT,
+                Collections.emptyMap(),
+                List.of(dataStreamName),
+                Collections.emptyList(),
+                System.currentTimeMillis(),
+                (long) randomIntBetween(0, 1000),
+                Map.of(),
+                null,
+                SnapshotInfoTestUtils.randomUserMetadata(),
+                IndexVersionUtils.randomVersion()
+            )
+        );
+        final DataStream dataStream = DataStreamTestHelper.randomInstance(dataStreamName);
+        ClusterState state = ClusterState.builder(clusterState(dataStream)).putCustom(SnapshotsInProgress.TYPE, snaps).build();
path: server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsServiceTests.java
user: login: JVerwolf
body: Ah, I should have read further down in the comments,I just spent some time figuring out how to do this after resolving your comment to remove the overloaded method!
created_at: 2025-03-13T15:54:14Z
author_association: CONTRIBUTOR
diff_hunk: @@ -432,6 +442,71 @@ private static IndexAbstraction validateIndex(ProjectMetadata project, String in
         return index;
     }
 
+    /**
+     * Removes the given data streams and their backing indices from the Cluster State.
+     *
+     * @param clusterState The cluster state
+     * @param dataStreams  The data streams to remove
+     * @param settings     The settings
+     * @return The updated Cluster State
+     */
+    @FixForMultiProject // Once callers have Project State we can update/remove this method.
+    public static ClusterState deleteDataStreams(ClusterState clusterState, Set<DataStream> dataStreams, Settings settings) {
path: server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java
user: login: nielsbauman
body: You can just call [`currentState.projectState()`](https://github.com/elastic/elasticsearch/blob/1bee2cc3ccbcb3bc917b9e88d91c69dfbe26ab02/server/src/main/java/org/elasticsearch/cluster/ClusterState.java#L536-L542). That basically does the same thing as you wrote, but less convoluted :)
created_at: 2025-03-13T17:51:24Z
author_association: CONTRIBUTOR
diff_hunk: @@ -466,6 +478,51 @@ public void testUpdateDataStreamOptions() {
         assertThat(updatedDataStream.getDataStreamOptions(), equalTo(DataStreamOptions.EMPTY));
     }
 
+    public void testDeleteMissing() {
+        DataStream dataStream = DataStreamTestHelper.randomInstance();
+        ClusterState state = ClusterState.builder(ClusterName.DEFAULT).build();
+        IllegalArgumentException e = expectThrows(
+            IllegalArgumentException.class,
+            () -> MetadataDataStreamsService.deleteDataStreams(state, Set.of(dataStream), Settings.EMPTY)
+        );
+        assertThat(e.getMessage(), containsString(dataStream.getName()));
+    }
+
+    public void testDeleteSnapshotting() {
+        String dataStreamName = randomAlphaOfLength(5);
+        Snapshot snapshot = new Snapshot("doesn't matter", new SnapshotId("snapshot name", "snapshot uuid"));
+        SnapshotsInProgress snaps = SnapshotsInProgress.EMPTY.withAddedEntry(
+            SnapshotsInProgress.Entry.snapshot(
+                snapshot,
+                true,
+                false,
+                SnapshotsInProgress.State.INIT,
+                Collections.emptyMap(),
+                List.of(dataStreamName),
+                Collections.emptyList(),
+                System.currentTimeMillis(),
+                (long) randomIntBetween(0, 1000),
+                Map.of(),
+                null,
+                SnapshotInfoTestUtils.randomUserMetadata(),
+                IndexVersionUtils.randomVersion()
+            )
+        );
+        final DataStream dataStream = DataStreamTestHelper.randomInstance(dataStreamName);
+        ClusterState state = ClusterState.builder(clusterState(dataStream)).putCustom(SnapshotsInProgress.TYPE, snaps).build();
path: server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsServiceTests.java
user: login: nielsbauman
body: Ah sorry about that, I assumed my comments would be read in the order I added them. I should have linked or referred to my other comment...
created_at: 2025-03-13T17:52:23Z
author_association: CONTRIBUTOR
diff_hunk: @@ -466,6 +479,56 @@ public void testUpdateDataStreamOptions() {
         assertThat(updatedDataStream.getDataStreamOptions(), equalTo(DataStreamOptions.EMPTY));
     }
 
+    public void testDeleteMissing() {
+        DataStream dataStream = DataStreamTestHelper.randomInstance();
+        ProjectState state = ClusterState.builder(ClusterName.DEFAULT).build().projectState(randomProjectIdOrDefault());
path: server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsServiceTests.java
user: login: nielsbauman
body: Since we're not adding a project to the cluster state and cluster states by default only have the `default` project, creating/extracting a project state with a random project ID is going to fail (about 50% of the time). The most correct way of doing it would be
```java
final var projectId = randomProjectIdOrDefault();
ProjectState state = ClusterState.builder(ClusterName.Default)
    .putProjectMetadata(ProjectMetadata.builder(projectId))
    .projectState(projectId);
```
That ensures the requested project is always present (but it can still be the default project).
created_at: 2025-03-13T17:57:46Z
author_association: CONTRIBUTOR
diff_hunk: @@ -466,6 +479,56 @@ public void testUpdateDataStreamOptions() {
         assertThat(updatedDataStream.getDataStreamOptions(), equalTo(DataStreamOptions.EMPTY));
     }
 
+    public void testDeleteMissing() {
+        DataStream dataStream = DataStreamTestHelper.randomInstance();
+        ProjectState state = ClusterState.builder(ClusterName.DEFAULT).build().projectState(randomProjectIdOrDefault());
path: server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsServiceTests.java
user: login: JVerwolf
body: Thanks @nielsbauman .  I had to modify slightly:
```java
        final var projectId = randomProjectIdOrDefault();
        ProjectState state = ClusterState.builder(ClusterName.DEFAULT)
            .putProjectMetadata(ProjectMetadata.builder(projectId))
            .build()
            .projectState(projectId);
```
I also changed the MetadataDataStreamsService#deleteDataStreams to throw a `ResourceNotFoundException` if the datastream doesn't exist. (the prior exception was thrown by the `projectFor` code that has since been removed.
created_at: 2025-03-13T18:37:03Z
author_association: CONTRIBUTOR
diff_hunk: @@ -608,6 +711,69 @@ public void testSystemIndexAliasesAreAlwaysRestored() {
 
     }
 
+    // TODO: This test is failing because the assertion `assertTrue(indexExists(systemDataStreamAlias));` trips.
+    @AwaitsFix(bugUrl = "assertTrue(indexExists(systemDataStreamAlias)); is broken")
path: server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemIndicesSnapshotIT.java
user: login: JVerwolf
body: Thanks @dakrone , I found the bug, fixed it, and enabled the test.
created_at: 2025-03-13T18:38:36Z
author_association: CONTRIBUTOR
path: server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsServiceTests.java
user: login: JVerwolf
body: Perhaps this is something we can do in a follow up PR? Ideally we'd have it in this one, but I'm not sure I'll have time to refactor all the testing before the RC cut.  I'll see what I can do.

 For now, our test coverage is still the same (transitively via the [DeleteDataStreamTransportActionTests.java](https://github.com/elastic/elasticsearch/blob/9cbb26247bad7ee50b3dcbea4ee64ed5deb14409/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/DeleteDataStreamTransportActionTests.java#L40-L40) tests).  I agree we should eventually move any coverage we can to this class as well.  
created_at: 2025-03-13T19:04:34Z
author_association: CONTRIBUTOR
diff_hunk: @@ -432,6 +440,52 @@ private static IndexAbstraction validateIndex(ProjectMetadata project, String in
         return index;
     }
 
+    /**
+     * Removes the given data stream and their backing indices from the Project State.
+     *
+     * @param projectState The project state
+     * @param dataStreams  The data streams to remove
+     * @param settings     The settings
+     * @return The updated Project State
+     */
+    public static ClusterState deleteDataStreams(ProjectState projectState, Set<DataStream> dataStreams, Settings settings) {
+        if (dataStreams.isEmpty()) {
+            return projectState.cluster();
+        }
+
+        Set<String> dataStreamNames = dataStreams.stream().map(DataStream::getName).collect(Collectors.toSet());
+        Set<String> snapshottingDataStreams = SnapshotsService.snapshottingDataStreams(projectState, dataStreamNames);
+        if (snapshottingDataStreams.isEmpty() == false) {
+            throw new SnapshotInProgressException(
+                "Cannot delete data streams that are being snapshotted: ["
+                    + String.join(", ", snapshottingDataStreams)
+                    + "]. Try again after snapshot finishes or cancel the currently running snapshot."
+            );
+        }
+
+        Set<Index> backingIndicesToRemove = new HashSet<>();
+        for (DataStream dataStream : dataStreams) {
+            assert dataStream != null;
+            backingIndicesToRemove.addAll(dataStream.getIndices());
+            backingIndicesToRemove.addAll(dataStream.getFailureIndices());
+        }
+
+        // first delete the data streams and then the indices:
+        // (this to avoid data stream validation from failing when deleting an index that is part of a data stream
+        // without updating the data stream)
+        // TODO: change order when "delete index api" also updates the data stream the "index to be removed" is a member of
+        ClusterState newState = projectState.updatedState(builder -> {
+            for (DataStream ds : dataStreams) {
+                LOGGER.info("removing data stream [{}]", ds.getName());
+                builder.removeDataStream(ds.getName());
+                if (projectState.metadata().dataStreams().get(ds.getName()) == null) {
+                    throw new ResourceNotFoundException("data stream [" + ds.getName() + "] not found");
+                }
path: server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java
user: login: nielsbauman
body: I think it would make more sense if we do this check in the loop where were constructing `backingIndicesToRemove`. Then we can exit early before building a new cluster state if a data stream doesn't exist. What do you think?
created_at: 2025-03-13T19:07:59Z
author_association: CONTRIBUTOR
diff_hunk: @@ -691,6 +898,7 @@ public void testNoneFeatureStateOnCreation() {
         assertThat(snapshottedIndices, allOf(hasItem(regularIndex), not(hasItem(SystemIndexTestPlugin.SYSTEM_INDEX_NAME))));
     }
 
+    // TODO, Do we need to test this for Datastreams?
path: server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemResourceSnapshotIT.java
user: login: JVerwolf
body: I'm not sure how we'd even set this up to test this.  Any ideas?
created_at: 2025-03-13T19:09:03Z
author_association: CONTRIBUTOR
diff_hunk: @@ -650,19 +652,21 @@ private static Tuple<Map<String, DataStream>, Map<String, DataStreamAlias>> getD
                         )
                     );
                 } else if (featureStateDataStreams.contains(requestedDataStream)) {
-                    dataStreams.put(requestedDataStream, dataStreamInSnapshot);
+                    allDataStreams.put(requestedDataStream, dataStreamInSnapshot);
+                    systemDataStreams.put(requestedDataStream, dataStreamInSnapshot);
                 } else {
                     logger.debug(
                         "omitting system data stream [{}] from snapshot restoration because its feature state was not requested",
                         requestedDataStream
                     );
                 }
             }
-            if (includeAliases) {
+            if (includeAliases || systemDataStreams.isEmpty() == false) {
                 dataStreamAliases = new HashMap<>();
                 final Map<String, DataStreamAlias> dataStreamAliasesInSnapshot = globalMetadata.getProject().dataStreamAliases();
+                Map<String, DataStream> dataStreamsWithAliases = includeAliases ? systemDataStreams : allDataStreams;
path: server/src/main/java/org/elasticsearch/snapshots/RestoreService.java
user: login: nielsbauman
body: Wouldn't this need to be inverted? So `includeAliases ? allDataStreams : systemDataStreams`?

Might also be worth adding a small comment to explain why we're doing that.
created_at: 2025-03-13T19:11:20Z
author_association: CONTRIBUTOR
path: server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsServiceTests.java
user: login: nielsbauman
body: I assumed it would be pretty straightforward to move the methods, but I definitely don't want to let this get in the way of you making your deadline, so a follow-up PR is fine with me :+1:.
created_at: 2025-03-13T19:13:49Z
author_association: CONTRIBUTOR
diff_hunk: @@ -691,6 +898,7 @@ public void testNoneFeatureStateOnCreation() {
         assertThat(snapshottedIndices, allOf(hasItem(regularIndex), not(hasItem(SystemIndexTestPlugin.SYSTEM_INDEX_NAME))));
     }
 
+    // TODO, Do we need to test this for Datastreams?
path: server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemResourceSnapshotIT.java
user: login: dakrone
body: A partial snapshot would be one where some of the shards were snapshotted and others weren't. If we take a snapshot with `partial: true` while the index is red (say it had two primaries and one were unavailable) that would cause a partial snapshot.

It would probably be good to test this as well, since we test it for regular system indices.
created_at: 2025-03-13T20:57:40Z
author_association: MEMBER
diff_hunk: @@ -691,6 +898,7 @@ public void testNoneFeatureStateOnCreation() {
         assertThat(snapshottedIndices, allOf(hasItem(regularIndex), not(hasItem(SystemIndexTestPlugin.SYSTEM_INDEX_NAME))));
     }
 
+    // TODO, Do we need to test this for Datastreams?
path: server/src/internalClusterTest/java/org/elasticsearch/snapshots/SystemResourceSnapshotIT.java
user: login: JVerwolf
body: Thanks @dakrone .  Fixed.
created_at: 2025-03-13T22:29:57Z
author_association: CONTRIBUTOR
diff_hunk: @@ -432,6 +440,52 @@ private static IndexAbstraction validateIndex(ProjectMetadata project, String in
         return index;
     }
 
+    /**
+     * Removes the given data stream and their backing indices from the Project State.
+     *
+     * @param projectState The project state
+     * @param dataStreams  The data streams to remove
+     * @param settings     The settings
+     * @return The updated Project State
+     */
+    public static ClusterState deleteDataStreams(ProjectState projectState, Set<DataStream> dataStreams, Settings settings) {
+        if (dataStreams.isEmpty()) {
+            return projectState.cluster();
+        }
+
+        Set<String> dataStreamNames = dataStreams.stream().map(DataStream::getName).collect(Collectors.toSet());
+        Set<String> snapshottingDataStreams = SnapshotsService.snapshottingDataStreams(projectState, dataStreamNames);
+        if (snapshottingDataStreams.isEmpty() == false) {
+            throw new SnapshotInProgressException(
+                "Cannot delete data streams that are being snapshotted: ["
+                    + String.join(", ", snapshottingDataStreams)
+                    + "]. Try again after snapshot finishes or cancel the currently running snapshot."
+            );
+        }
+
+        Set<Index> backingIndicesToRemove = new HashSet<>();
+        for (DataStream dataStream : dataStreams) {
+            assert dataStream != null;
+            if (projectState.metadata().dataStreams().get(dataStream.getName()) == null) {
+                throw new ResourceNotFoundException("data stream [" + dataStream.getName() + "] not found");
+            }
+            backingIndicesToRemove.addAll(dataStream.getIndices());
+            backingIndicesToRemove.addAll(dataStream.getFailureIndices());
+        }
+
+        // first delete the data streams and then the indices:
+        // (this to avoid data stream validation from failing when deleting an index that is part of a data stream
+        // without updating the data stream)
+        // TODO: change order when "delete index api" also updates the data stream the "index to be removed" is a member of
path: server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java
user: login: dakrone
body: I don't understand this TODO entirely, why would we want to change the order when invoked by the delete index API?
created_at: 2025-03-13T22:43:35Z
author_association: MEMBER
diff_hunk: @@ -432,6 +440,52 @@ private static IndexAbstraction validateIndex(ProjectMetadata project, String in
         return index;
     }
 
+    /**
+     * Removes the given data stream and their backing indices from the Project State.
+     *
+     * @param projectState The project state
+     * @param dataStreams  The data streams to remove
+     * @param settings     The settings
+     * @return The updated Project State
+     */
+    public static ClusterState deleteDataStreams(ProjectState projectState, Set<DataStream> dataStreams, Settings settings) {
+        if (dataStreams.isEmpty()) {
+            return projectState.cluster();
+        }
+
+        Set<String> dataStreamNames = dataStreams.stream().map(DataStream::getName).collect(Collectors.toSet());
+        Set<String> snapshottingDataStreams = SnapshotsService.snapshottingDataStreams(projectState, dataStreamNames);
+        if (snapshottingDataStreams.isEmpty() == false) {
+            throw new SnapshotInProgressException(
+                "Cannot delete data streams that are being snapshotted: ["
+                    + String.join(", ", snapshottingDataStreams)
+                    + "]. Try again after snapshot finishes or cancel the currently running snapshot."
+            );
+        }
+
+        Set<Index> backingIndicesToRemove = new HashSet<>();
+        for (DataStream dataStream : dataStreams) {
+            assert dataStream != null;
+            if (projectState.metadata().dataStreams().get(dataStream.getName()) == null) {
+                throw new ResourceNotFoundException("data stream [" + dataStream.getName() + "] not found");
+            }
+            backingIndicesToRemove.addAll(dataStream.getIndices());
+            backingIndicesToRemove.addAll(dataStream.getFailureIndices());
+        }
+
+        // first delete the data streams and then the indices:
+        // (this to avoid data stream validation from failing when deleting an index that is part of a data stream
+        // without updating the data stream)
+        // TODO: change order when "delete index api" also updates the data stream the "index to be removed" is a member of
path: server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java
user: login: JVerwolf
body: Me neither, but it was already there in the transport action so I kept it. 
created_at: 2025-03-13T22:58:22Z
author_association: CONTRIBUTOR

PR comments:
user: login: elasticsearchmachine
created_at: 2025-03-12T14:31:08Z
body: Hi @JVerwolf, I've created a changelog YAML for you.
author_association: COLLABORATOR
user: login: elasticsearchmachine
created_at: 2025-03-13T05:26:51Z
body: Pinging @elastic/es-core-infra (Team:Core/Infra)
author_association: COLLABORATOR
user: login: elasticsearchmachine
created_at: 2025-03-13T18:00:21Z
body: Pinging @elastic/es-data-management (Team:Data Management)
author_association: COLLABORATOR
user: login: elasticsearchmachine
created_at: 2025-03-14T15:02:15Z
body: ## 💔 Backport failed
| Status | Branch | Result |
|:------:|:------:|:------:|
| ❌ |  8.18  | Commit could not be cherrypicked due to conflicts |
| ❌ |  8.x  | Commit could not be cherrypicked due to conflicts |
| ❌ |  9.0  | Commit could not be cherrypicked due to conflicts |

You can use [sqren/backport](https://github.com/sqren/backport) to manually backport by running `backport --upstream elastic/elasticsearch --pr 124651`
author_association: COLLABORATOR
user: login: JVerwolf
created_at: 2025-03-14T18:01:14Z
body: ## 💔 Some backports could not be created

| Status | Branch | Result |
|:------:|:------:|:------|
|✅|8.x|[<img src="https://img.shields.io/github/pulls/detail/state/elastic/elasticsearch/124900">](https://github.com/elastic/elasticsearch/pull/124900)|
|✅|9.0|[<img src="https://img.shields.io/github/pulls/detail/state/elastic/elasticsearch/124910">](https://github.com/elastic/elasticsearch/pull/124910)|
|❌|8.18|An unhandled error occurred. Please see the logs for details|

### Manual backport
To create the backport manually run:
```
backport --pr 124651
```

### Questions ?
Please refer to the [Backport tool documentation](https://github.com/sorenlouv/backport)

<!--- Backport version: 9.6.6 -->
author_association: CONTRIBUTOR
user: login: JVerwolf
created_at: 2025-03-14T18:26:46Z
body: ## 💚 All backports created successfully

| Status | Branch | Result |
|:------:|:------:|:------|
|✅|8.18|[<img src="https://img.shields.io/github/pulls/detail/state/elastic/elasticsearch/124912">](https://github.com/elastic/elasticsearch/pull/124912)|

### Questions ?
Please refer to the [Backport tool documentation](https://github.com/sorenlouv/backport)

<!--- Backport version: 9.6.6 -->
author_association: CONTRIBUTOR
user: login: JVerwolf
created_at: 2025-03-14T18:30:28Z
body: Curiously, the v8 backports are all failing the `SystemResourceSnapshotTests` (previously `SystemIndexSnapshotTests`) when the whole test class is run, but they all succeed when run individually. In a debugger, I see missing features that should have been added by the plugins.  Seems like a test framework bug.  Does anyone here have intuition about what might have changed between the 8/9 versions?
author_association: CONTRIBUTOR