Repository: dadoonet/elasticsearch-beyonder (java)

Issue:
Manage rollover and date maths indices like `<logstash-{now/d-2d}-000001>`
Based on [discussion](https://github.com/dadoonet/elasticsearch-beyonder/issues/327#issuecomment-2698942355) in #327:

> when an index already exists with a previous date but also with another suffix than -000001, I wish that Beyonder does not create the index. My aim is only to create the first index to start ILM mechanism. Then, Elasticsearch takes over and manages index lifecycle.

ILM is using rollover behind the scene which can lead to indices from `index-000001` to be transformed to `index-000002`.
When the ILM has a DELETE phase, it might remove at some point the `index-000001` index.

And when we start again Beyonder, no new index should be created as we do have an index like `index-000002`.

Adding even more complexity, and index name can also use [date maths](https://www.elastic.co/guide/en/elasticsearch/reference/current/api-conventions.html#api-date-math-index-names), like `<logstash-{now/d-2d}-000001>`.

The index name might be first computed as `logstash-2025.03.03-000001` but then rolled over as `logstash-2025.03.03-000002`. After a day, it will become something like `logstash-2025.03.04-000001` and when rolled over: `logstash-2025.03.04-000002`.

This means that when we check if an index already exists, we need to transform some of the characters before checking if any index matches the wildcard:

* `logstash` we check `logstash`
* `logstash-000001` we check `logstash-*`
* `<logstash-{now/d-2d}>` we check `logstash-*`
* `<logstash-{now/d-2d}-000001>` we check `logstash-*-*`


Pull request:
URL Decode index names before replacing with wildcards
On disk, dirs might be URL encoded when using date maths indices. For example `<my-index-{now/d}-000001>` would be on written disk as `%3Cmy-index-%7Bnow%2Fd%7D-000001%3E`.

We need to first URL decode the filename before checking if we need to replace the index name with wildcards.

Closes #354.

Patch:
diff --git a/src/main/java/fr/pilato/elasticsearch/tools/util/ResourceList.java b/src/main/java/fr/pilato/elasticsearch/tools/util/ResourceList.java
index bc6e0aa..3702d39 100644
--- a/src/main/java/fr/pilato/elasticsearch/tools/util/ResourceList.java
+++ b/src/main/java/fr/pilato/elasticsearch/tools/util/ResourceList.java
@@ -24,6 +24,7 @@ import org.slf4j.LoggerFactory;
 
 import java.io.File;
 import java.io.IOException;
+import java.io.UnsupportedEncodingException;
 import java.net.URISyntaxException;
 import java.net.URL;
 import java.net.URLDecoder;
@@ -272,17 +273,29 @@ public class ResourceList {
     }
 
     /**
-     * Replace index name from a form of "<my-index-{now/d}-000001>" to "my-index-{now/d}-*"
+     * Replace index name from a form of "<my-index-{now/d}-000001>" or "%3Cmy-index-%7Bnow%2Fd%7D-000001%3E"
+     * to "my-index-*-*".
      * @param indexName the index name to replace
      * @return the replaced index name
      */
-    public static String replaceIndexName(String indexName) {
+    public static String replaceIndexName(final String indexName) {
         logger.trace("replaceIndexName({})", indexName);
-        String replaced = indexName
-                .replaceAll("\\{[^}]*\\}", "*")
-                .replaceAll("<([^>]*)>", "$1")
-                .replaceAll("-\\d{6}", "-*");
-        logger.trace("/replaceIndexName({}) = [{}]", indexName, replaced);
-        return replaced;
+        try {
+            String replaced =
+                    // We need to first URL decode the index name
+                    URLDecoder.decode(indexName, "UTF8")
+                    // We replace {WHATEVER} with *
+                    .replaceAll("\\{[^}]*\\}", "*")
+                    // We replace <WHATEVER> with WHATEVER
+                    .replaceAll("<([^>]*)>", "$1")
+                    // We replace the six ending digits like -123456 with -*
+                    .replaceAll("-\\d{6}", "-*");
+            logger.trace("/replaceIndexName({}) = [{}]", indexName, replaced);
+            return replaced;
+        } catch (UnsupportedEncodingException e) {
+            logger.warn("Exception raised while URL decoding [{}]: {}. We return the original value.",
+                    indexName, e.getMessage());
+            return indexName;
+        }
     }
 }


Test patch:
diff --git a/src/test/java/fr/pilato/elasticsearch/tools/BeyonderRestIT.java b/src/test/java/fr/pilato/elasticsearch/tools/BeyonderRestIT.java
index 2f696ea..b2ccc99 100644
--- a/src/test/java/fr/pilato/elasticsearch/tools/BeyonderRestIT.java
+++ b/src/test/java/fr/pilato/elasticsearch/tools/BeyonderRestIT.java
@@ -625,11 +625,20 @@ public class BeyonderRestIT extends AbstractBeyonderTest {
         // Refresh the indices
         client.performRequest(new Request("POST", "/_refresh"));
 
-        // Check that we have 10 documents in my-index-* index because the daily index was not existing yet but only
-        // the yesterday index
-        Map<String, Object> response = asMap(client.performRequest(new Request("GET", "/my-index-*/_search")));
-        String numberOfHits = BeanUtils.getProperty(response, "hits.total.value");
-        assertThat(numberOfHits, equalTo("10"));
+        // Expected outcome is:
+        // Index from yesterday should exist but with no documents
+        {
+            Map<String, Object> response = asMap(client.performRequest(new Request("GET", "/%3Cmy-index-%7Bnow%2Fd-1d%7D%3E/_search")));
+            String numberOfHits = BeanUtils.getProperty(response, "hits.total.value");
+            assertThat(numberOfHits, equalTo("0"));
+        }
+
+        // Index from today should not have been created as we do have yet an older index with the same date math
+        {
+            Map<String, Object> response = asMap(client.performRequest(new Request("GET", "/my-index-*/_search")));
+            String numberOfHits = BeanUtils.getProperty(response, "hits.total.value");
+            assertThat(numberOfHits, equalTo("0"));
+        }
     }
 
     @Test
diff --git a/src/test/java/fr/pilato/elasticsearch/tools/ResourceListTest.java b/src/test/java/fr/pilato/elasticsearch/tools/ResourceListTest.java
index 9b27240..d0975fb 100644
--- a/src/test/java/fr/pilato/elasticsearch/tools/ResourceListTest.java
+++ b/src/test/java/fr/pilato/elasticsearch/tools/ResourceListTest.java
@@ -53,9 +53,21 @@ public class ResourceListTest {
 
     @Test
     public void testIndexNames() {
+        // We test simple index names
         assertThat(replaceIndexName("foo"), is("foo"));
+        assertThat(replaceIndexName("foo-001"), is("foo-001"));
+        assertThat(replaceIndexName("000001-foo"), is("000001-foo"));
+
+        // We test rollover indices
         assertThat(replaceIndexName("my-index-000001"), is("my-index-*"));
+        assertThat(replaceIndexName("my-index-001234"), is("my-index-*"));
+
+        // We test date maths
         assertThat(replaceIndexName("<my-index-{now/d}>"), is("my-index-*"));
         assertThat(replaceIndexName("<my-index-{now/d}-000001>"), is("my-index-*-*"));
+
+        // The same but with URL encoded characters
+        assertThat(replaceIndexName("%3Cmy-index-%7Bnow%2Fd%7D%3E"), is("my-index-*"));
+        assertThat(replaceIndexName("%3Cmy-index-%7Bnow%2Fd%7D-000001%3E"), is("my-index-*-*"));
     }
 }