Repository: dadoonet/elasticsearch-beyonder (java)

Issue:
Manage rollover and date maths indices like `<logstash-{now/d-2d}-000001>`
Based on [discussion](https://github.com/dadoonet/elasticsearch-beyonder/issues/327#issuecomment-2698942355) in #327:

> when an index already exists with a previous date but also with another suffix than -000001, I wish that Beyonder does not create the index. My aim is only to create the first index to start ILM mechanism. Then, Elasticsearch takes over and manages index lifecycle.

ILM is using rollover behind the scene which can lead to indices from `index-000001` to be transformed to `index-000002`.
When the ILM has a DELETE phase, it might remove at some point the `index-000001` index.

And when we start again Beyonder, no new index should be created as we do have an index like `index-000002`.

Adding even more complexity, and index name can also use [date maths](https://www.elastic.co/guide/en/elasticsearch/reference/current/api-conventions.html#api-date-math-index-names), like `<logstash-{now/d-2d}-000001>`.

The index name might be first computed as `logstash-2025.03.03-000001` but then rolled over as `logstash-2025.03.03-000002`. After a day, it will become something like `logstash-2025.03.04-000001` and when rolled over: `logstash-2025.03.04-000002`.

This means that when we check if an index already exists, we need to transform some of the characters before checking if any index matches the wildcard:

* `logstash` we check `logstash`
* `logstash-000001` we check `logstash-*`
* `<logstash-{now/d-2d}>` we check `logstash-*`
* `<logstash-{now/d-2d}-000001>` we check `logstash-*-*`


Pull request description:
Manage rollover and date maths indices like `<logstash-{now/d-2d}-000001>`
ILM is using rollover behind the scene which can lead to indices from `index-000001` to be transformed to `index-000002`.
When the ILM has a DELETE phase, it might remove at some point the `index-000001` index.

And when we start again Beyonder, no new index should be created as we do have an index like `index-000002`.

Adding even more complexity, and index name can also use [date maths](https://www.elastic.co/guide/en/elasticsearch/reference/current/api-conventions.html#api-date-math-index-names), like `<logstash-{now/d-2d}-000001>`.

The index name might be first computed as `logstash-2025.03.03-000001` but then rolled over as `logstash-2025.03.03-000002`. After a day, it will become something like `logstash-2025.03.04-000001` and when rolled over: `logstash-2025.03.04-000002`.

This means that when we check if an index already exists, we need to transform some of the characters before checking if any index matches the wildcard:

* `logstash` we check `logstash`
* `logstash-000001` we check `logstash-*`
* `<logstash-{now/d-2d}>` we check `logstash-*`
* `<logstash-{now/d-2d}-000001>` we check `logstash-*-*`

Closes #354

Patch:
diff --git a/src/main/java/fr/pilato/elasticsearch/tools/updaters/ElasticsearchAliasUpdater.java b/src/main/java/fr/pilato/elasticsearch/tools/updaters/ElasticsearchAliasUpdater.java
index 05cec68..da587f0 100644
--- a/src/main/java/fr/pilato/elasticsearch/tools/updaters/ElasticsearchAliasUpdater.java
+++ b/src/main/java/fr/pilato/elasticsearch/tools/updaters/ElasticsearchAliasUpdater.java
@@ -33,7 +33,7 @@ public class ElasticsearchAliasUpdater {
     private static final Logger logger = LoggerFactory.getLogger(ElasticsearchAliasUpdater.class);
 
     /**
-     * Create a component template in Elasticsearch.
+     * Manage global aliases in Elasticsearch.
      * @param client Elasticsearch client
      * @param root dir within the classpath
      * @throws Exception if something goes wrong
@@ -52,7 +52,7 @@ public class ElasticsearchAliasUpdater {
      * @param json JSon content for the aliases
      * @throws Exception if something goes wrong
      */
-    public static void manageAliasesWithJsonInElasticsearch(RestClient client, String json) throws Exception {
+    private static void manageAliasesWithJsonInElasticsearch(RestClient client, String json) throws Exception {
         logger.trace("manageAliases()");
 
         assert client != null;
diff --git a/src/main/java/fr/pilato/elasticsearch/tools/updaters/ElasticsearchIndexUpdater.java b/src/main/java/fr/pilato/elasticsearch/tools/updaters/ElasticsearchIndexUpdater.java
index 7884cd6..02af724 100644
--- a/src/main/java/fr/pilato/elasticsearch/tools/updaters/ElasticsearchIndexUpdater.java
+++ b/src/main/java/fr/pilato/elasticsearch/tools/updaters/ElasticsearchIndexUpdater.java
@@ -27,6 +27,10 @@ import org.elasticsearch.client.RestClient;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+
+import static fr.pilato.elasticsearch.tools.util.ResourceList.replaceIndexName;
 import static fr.pilato.elasticsearch.tools.util.SettingsReader.getJsonContent;
 
 /**
@@ -59,7 +63,7 @@ public class ElasticsearchIndexUpdater {
 	 * @return true if we created the index and false if the index already existed
 	 * @throws Exception if the elasticsearch API call is failing
 	 */
-	public static boolean createIndexWithSettings(RestClient client, String index, String settings, boolean force) throws Exception {
+	private static boolean createIndexWithSettings(RestClient client, String index, String settings, boolean force) throws Exception {
 		if (force && isIndexExist(client, index)) {
 			logger.debug("Index [{}] already exists but force set to true. Removing all data!", index);
 			removeIndexInElasticsearch(client, index);
@@ -80,7 +84,7 @@ public class ElasticsearchIndexUpdater {
 	 * @param index Index name
 	 * @throws Exception if the elasticsearch API call is failing
 	 */
-	public static void removeIndexInElasticsearch(RestClient client, String index) throws Exception {
+	private static void removeIndexInElasticsearch(RestClient client, String index) throws Exception {
 		logger.trace("removeIndex([{}])", index);
 
 		assert client != null;
@@ -189,9 +193,23 @@ public class ElasticsearchIndexUpdater {
 	 * @return true if index already exists
 	 * @throws Exception if the elasticsearch API call is failing
 	 */
-	public static boolean isIndexExist(RestClient client, String index) throws Exception {
-		Response response = client.performRequest(new Request("HEAD", "/" + index));
-		return response.getStatusLine().getStatusCode() == 200;
+	public static boolean isIndexExist(RestClient client, final String index) throws Exception {
+		try {
+			Response response = client.performRequest(new Request("GET", "/" + replaceIndexName(index)));
+
+			// Read the response as a String
+			String responseBody = new BufferedReader(new InputStreamReader(response.getEntity().getContent()))
+					.lines().reduce("", (accumulator, actual) -> accumulator + actual);
+			logger.trace("{}", responseBody);
+
+			// If we don't have an empty response ("{}"), then at least one index exists with the pattern
+			return !"{}".equals(responseBody);
+		} catch (ResponseException e) {
+			if (e.getResponse().getStatusLine().getStatusCode() == 404) {
+				return false;
+			}
+			throw e;
+		}
 	}
 
 	/**
diff --git a/src/main/java/fr/pilato/elasticsearch/tools/util/ResourceList.java b/src/main/java/fr/pilato/elasticsearch/tools/util/ResourceList.java
index 540b147..bc6e0aa 100644
--- a/src/main/java/fr/pilato/elasticsearch/tools/util/ResourceList.java
+++ b/src/main/java/fr/pilato/elasticsearch/tools/util/ResourceList.java
@@ -270,4 +270,19 @@ public class ResourceList {
 
         return sortedFilenames;
     }
+
+    /**
+     * Replace index name from a form of "<my-index-{now/d}-000001>" to "my-index-{now/d}-*"
+     * @param indexName the index name to replace
+     * @return the replaced index name
+     */
+    public static String replaceIndexName(String indexName) {
+        logger.trace("replaceIndexName({})", indexName);
+        String replaced = indexName
+                .replaceAll("\\{[^}]*\\}", "*")
+                .replaceAll("<([^>]*)>", "$1")
+                .replaceAll("-\\d{6}", "-*");
+        logger.trace("/replaceIndexName({}) = [{}]", indexName, replaced);
+        return replaced;
+    }
 }


Test patch:
diff --git a/src/test/java/fr/pilato/elasticsearch/tools/BeyonderRestIT.java b/src/test/java/fr/pilato/elasticsearch/tools/BeyonderRestIT.java
index 4229bf8..e6dacab 100644
--- a/src/test/java/fr/pilato/elasticsearch/tools/BeyonderRestIT.java
+++ b/src/test/java/fr/pilato/elasticsearch/tools/BeyonderRestIT.java
@@ -31,7 +31,6 @@ import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.client.RestClientBuilder;
-import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
@@ -39,8 +38,10 @@ import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.BufferedReader;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.io.InputStreamReader;
 import java.net.ConnectException;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -149,7 +150,7 @@ public class BeyonderRestIT extends AbstractBeyonderTest {
         }
     }
 
-    @Before @After
+    @Before // @After
     public void cleanCluster() {
         // DELETE /twitter
         launchAndIgnoreFailure(() -> client.performRequest(new Request("DELETE", "/twitter")));
@@ -157,8 +158,10 @@ public class BeyonderRestIT extends AbstractBeyonderTest {
         launchAndIgnoreFailure(() -> client.performRequest(new Request("DELETE", "/test_*")));
         // DELETE /person
         launchAndIgnoreFailure(() -> client.performRequest(new Request("DELETE", "/person")));
-        // DELETE /person
+        // DELETE /my-index-*
         launchAndIgnoreFailure(() -> client.performRequest(new Request("DELETE", "/my-index-*")));
+        // DELETE /timeseries-*
+        launchAndIgnoreFailure(() -> client.performRequest(new Request("DELETE", "/timeseries-*")));
 
         // DELETE /_ingest/pipeline/twitter_pipeline
         launchAndIgnoreFailure(() -> client.performRequest(new Request("DELETE", "/_ingest/pipeline/twitter_pipeline")));
@@ -632,6 +635,111 @@ public class BeyonderRestIT extends AbstractBeyonderTest {
         assertThat(numberOfHits, equalTo("10"));
     }
 
+    @Test
+    public void testRolloverSimple() throws Exception {
+        testBeyonder("models/rollover-simple",
+                singletonList("timeseries-000001"),
+                null, null, null, null);
+
+        // Manually trigger the rollover
+        client.performRequest(new Request("POST", "/timeseries/_rollover"));
+
+        // Post another document to trigger rollover
+        Request postDoc = new Request("POST", "/timeseries/_doc");
+        postDoc.setJsonEntity("{\"message\":\"message 2\"}");
+        client.performRequest(postDoc);
+
+        // Refresh the indices
+        client.performRequest(new Request("POST", "/_refresh"));
+
+        // Check that we have 2 documents in timeseries index
+        {
+            Map<String, Object> response = asMap(client.performRequest(new Request("GET", "/timeseries/_search")));
+            String numberOfHits = BeanUtils.getProperty(response, "hits.total.value");
+            assertThat(numberOfHits, equalTo("2"));
+        }
+
+        // Delete the first index to simulate an ILM DELETE phase
+        client.performRequest(new Request("DELETE", "/timeseries-000001"));
+
+        // Check that we have 1 document in timeseries index
+        {
+            Map<String, Object> response = asMap(client.performRequest(new Request("GET", "/timeseries/_search")));
+            String numberOfHits = BeanUtils.getProperty(response, "hits.total.value");
+            assertThat(numberOfHits, equalTo("1"));
+        }
+
+        // Call Beyonder again and it should not fail
+        testBeyonder("models/rollover-simple",
+                singletonList("timeseries-000002"),
+                null, null, null, null);
+
+        // We should not have the timeseries-000001 index
+        assertThat(client.performRequest(new Request("HEAD", "/timeseries-000001"))
+                .getStatusLine().getStatusCode(), not(200));
+
+        // Check that we still have only 1 document in timeseries index
+        {
+            Map<String, Object> response = asMap(client.performRequest(new Request("GET", "/timeseries/_search")));
+            String numberOfHits = BeanUtils.getProperty(response, "hits.total.value");
+            assertThat(numberOfHits, equalTo("1"));
+        }
+    }
+
+    @Test
+    public void testRolloverWithDateMaths() throws Exception {
+        testBeyonder("models/rollover-date-maths",
+                singletonList("timeseries-*-000001"),
+                null, null, null, null);
+
+        // Manually trigger the rollover
+        client.performRequest(new Request("POST", "/timeseries/_rollover"));
+
+        // Post another document to trigger rollover
+        Request postDoc = new Request("POST", "/timeseries/_doc");
+        postDoc.setJsonEntity("{\"message\":\"message 2\"}");
+        client.performRequest(postDoc);
+
+        // Refresh the indices
+        client.performRequest(new Request("POST", "/_refresh"));
+
+        // Check that we have 2 documents in timeseries index
+        {
+            Map<String, Object> response = asMap(client.performRequest(new Request("GET", "/timeseries/_search")));
+            String numberOfHits = BeanUtils.getProperty(response, "hits.total.value");
+            assertThat(numberOfHits, equalTo("2"));
+        }
+
+        // Delete the first index to simulate an ILM DELETE phase
+        client.performRequest(new Request("DELETE", "/timeseries-*-000001"));
+
+        // Check that we have 1 document in timeseries index
+        {
+            Map<String, Object> response = asMap(client.performRequest(new Request("GET", "/timeseries/_search")));
+            String numberOfHits = BeanUtils.getProperty(response, "hits.total.value");
+            assertThat(numberOfHits, equalTo("1"));
+        }
+
+        // Call Beyonder again and it should not fail
+        testBeyonder("models/rollover-simple",
+                singletonList("timeseries-*-000002"),
+                null, null, null, null);
+
+        // We should not have the timeseries-*-000001 index
+        String responseBody = new BufferedReader(new InputStreamReader(
+                client.performRequest(new Request("GET", "/timeseries-*-000001")).getEntity().getContent()))
+                .lines()
+                .reduce("", (accumulator, actual) -> accumulator + actual);
+        assertThat(responseBody, is("{}"));
+
+        // Check that we still have only 1 document in timeseries index
+        {
+            Map<String, Object> response = asMap(client.performRequest(new Request("GET", "/timeseries/_search")));
+            String numberOfHits = BeanUtils.getProperty(response, "hits.total.value");
+            assertThat(numberOfHits, equalTo("1"));
+        }
+    }
+
     private String getMapping(String indexName) throws IOException {
         HttpEntity response = client.performRequest(new Request("GET", indexName + "/_mapping")).getEntity();
         ByteArrayOutputStream out = new ByteArrayOutputStream(Math.toIntExact(response.getContentLength() > 0 ? response.getContentLength() : 4000L));
diff --git a/src/test/java/fr/pilato/elasticsearch/tools/ResourceListTest.java b/src/test/java/fr/pilato/elasticsearch/tools/ResourceListTest.java
index 9697311..9b27240 100644
--- a/src/test/java/fr/pilato/elasticsearch/tools/ResourceListTest.java
+++ b/src/test/java/fr/pilato/elasticsearch/tools/ResourceListTest.java
@@ -27,6 +27,7 @@ import java.io.IOException;
 import java.net.URISyntaxException;
 import java.util.regex.Pattern;
 
+import static fr.pilato.elasticsearch.tools.util.ResourceList.replaceIndexName;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.is;
 
@@ -48,6 +49,13 @@ public class ResourceListTest {
         String filename = "/Users/dpilato/Documents/Elasticsearch/dev/spring-elasticsearch/target/test-classes/es/twitter/_settings.json";
         Pattern pattern = Pattern.compile(".*/" + root + "/.*");
         assertThat(pattern.matcher(filename).matches(), is(true));
+    }
 
+    @Test
+    public void testIndexNames() {
+        assertThat(replaceIndexName("foo"), is("foo"));
+        assertThat(replaceIndexName("my-index-000001"), is("my-index-*"));
+        assertThat(replaceIndexName("<my-index-{now/d}>"), is("my-index-*"));
+        assertThat(replaceIndexName("<my-index-{now/d}-000001>"), is("my-index-*-*"));
     }
 }
diff --git a/src/test/resources/models/rollover-date-maths/%3Ctimeseries-%7Bnow%2Fd-2d%7D-000001%3E/_data/doc1.json b/src/test/resources/models/rollover-date-maths/%3Ctimeseries-%7Bnow%2Fd-2d%7D-000001%3E/_data/doc1.json
new file mode 100644
index 0000000..2d20969
--- /dev/null
+++ b/src/test/resources/models/rollover-date-maths/%3Ctimeseries-%7Bnow%2Fd-2d%7D-000001%3E/_data/doc1.json
@@ -0,0 +1,3 @@
+{
+  "message": "message 1"
+}
diff --git a/src/test/resources/models/rollover-date-maths/%3Ctimeseries-%7Bnow%2Fd-2d%7D-000001%3E/_settings.json b/src/test/resources/models/rollover-date-maths/%3Ctimeseries-%7Bnow%2Fd-2d%7D-000001%3E/_settings.json
new file mode 100644
index 0000000..007f8b7
--- /dev/null
+++ b/src/test/resources/models/rollover-date-maths/%3Ctimeseries-%7Bnow%2Fd-2d%7D-000001%3E/_settings.json
@@ -0,0 +1,7 @@
+{
+  "aliases": {
+    "timeseries": {
+      "is_write_index": true
+    }
+  }
+}
diff --git a/src/test/resources/models/rollover-simple/timeseries-000001/_data/doc1.json b/src/test/resources/models/rollover-simple/timeseries-000001/_data/doc1.json
new file mode 100644
index 0000000..2d20969
--- /dev/null
+++ b/src/test/resources/models/rollover-simple/timeseries-000001/_data/doc1.json
@@ -0,0 +1,3 @@
+{
+  "message": "message 1"
+}
diff --git a/src/test/resources/models/rollover-simple/timeseries-000001/_settings.json b/src/test/resources/models/rollover-simple/timeseries-000001/_settings.json
new file mode 100644
index 0000000..007f8b7
--- /dev/null
+++ b/src/test/resources/models/rollover-simple/timeseries-000001/_settings.json
@@ -0,0 +1,7 @@
+{
+  "aliases": {
+    "timeseries": {
+      "is_write_index": true
+    }
+  }
+}