Repository: dadoonet/elasticsearch-beyonder (java)

Issue:
Rename `_pipeline` directory to `_pipelines`

Pull request description:
Rename `_pipeline` directory to `_pipelines`
We are making things more consistent:

* `_pipeline` dir has been deprecated by `_pipelines` dir.
* `force` parameter is not applied anymore to pipelines. So pipelines are always updated.

Closes #168.

Patch:
diff --git a/README.md b/README.md
index c80df1d..f5b6e58 100644
--- a/README.md
+++ b/README.md
@@ -44,6 +44,15 @@ Build Status
 [![Build Status](https://github.com/dadoonet/elasticsearch-beyonder/actions/workflows/maven.yml/badge.svg)](https://github.com/dadoonet/elasticsearch-beyonder/actions/workflows/maven.yml)
 
 
+Release notes
+=============
+
+7.13.1
+------
+
+* `_pipeline` dir has been deprecated by `_pipelines` dir.
+* `force` parameter is not applied anymore to pipelines. So pipelines are always updated.
+
 Getting Started
 ===============
 
@@ -343,7 +352,7 @@ Managing pipelines
 A pipeline is a definition of a series of processors that are to be executed in the same order as they are declared while 
 documents are being indexed. Please note that this feature is only supported when you use the REST client not the Transport client.
 
-For example, setting one fields value based on another field by using an Set Processor you an add a file named `elasticsearch/_pipeline/set_field_processor`
+For example, setting one fields value based on another field by using an Set Processor you can add a file named `elasticsearch/_pipelines/set_field_processor.json`
 in your project:
 
 ```json
@@ -360,11 +369,6 @@ in your project:
 }
 ```
 
-By default, Beyonder will not overwrite a pipeline if it already exists.
-This can be overridden by setting `force` to `true` in the expanded factory method
-`ElasticsearchBeyonder.start()`.
-
-
 # Tests
 
 This project comes with unit tests and integration tests.
diff --git a/src/main/java/fr/pilato/elasticsearch/tools/ElasticsearchBeyonder.java b/src/main/java/fr/pilato/elasticsearch/tools/ElasticsearchBeyonder.java
index ab9f423..4f1d0e5 100644
--- a/src/main/java/fr/pilato/elasticsearch/tools/ElasticsearchBeyonder.java
+++ b/src/main/java/fr/pilato/elasticsearch/tools/ElasticsearchBeyonder.java
@@ -127,9 +127,16 @@ public class ElasticsearchBeyonder {
 		}
 
 		// create pipelines
-		List<String> pipelineNames = ResourceList.getResourceNames(root, Defaults.PipelineDir);
+		List<String> pipelineNames = ResourceList.getResourceNames(root, Defaults.PipelinesDir);
+		if (pipelineNames.isEmpty()) {
+			// We check for deprecated resources
+			pipelineNames = ResourceList.getResourceNames(root, Defaults.PipelineDir);
+			if (!pipelineNames.isEmpty()) {
+				logger.warn("{} dir has been deprecated. Please use {} dir instead.", Defaults.PipelineDir, Defaults.PipelinesDir);
+			}
+		}
 		for (String pipelineName : pipelineNames) {
-			createPipeline(client, root, pipelineName, force);
+			createPipeline(client, root, pipelineName);
 		}
 
 		// create indices
diff --git a/src/main/java/fr/pilato/elasticsearch/tools/updaters/ElasticsearchPipelineUpdater.java b/src/main/java/fr/pilato/elasticsearch/tools/updaters/ElasticsearchPipelineUpdater.java
index a86cf00..fbeff60 100644
--- a/src/main/java/fr/pilato/elasticsearch/tools/updaters/ElasticsearchPipelineUpdater.java
+++ b/src/main/java/fr/pilato/elasticsearch/tools/updaters/ElasticsearchPipelineUpdater.java
@@ -47,38 +47,17 @@ public class ElasticsearchPipelineUpdater {
      * @param client Elasticsearch client
      * @param root dir within the classpath
      * @param pipeline the id of the pipeline
-     * @param force set it to true if you want to force cleaning pipeline before adding it
      * @throws Exception if something goes wrong
      */
-    public static void createPipeline(RestClient client, String root, String pipeline, boolean force) throws Exception {
-        String json = getJsonContent(root, SettingsFinder.Defaults.PipelineDir, pipeline);
-        createPipelineWithJson(client, pipeline, json, force);
-    }
-
-    /**
-     * Create a new pipeline in Elasticsearch
-     * 
-     * @param client Elasticsearch client
-     * @param pipeline the id of the pipeline
-     * @param json JSon content for the pipeline
-     * @param force set it to true if you want to force cleaning pipeline before adding it
-     * @throws Exception if something goes wrong
-     */
-    public static void createPipelineWithJson(RestClient client, String pipeline, String json, boolean force) throws Exception {
-        if (isPipelineExist(client, pipeline)) {
-            if (force) {
-                logger.debug("Pipeline [{}] already exists. Force is set. Overriding it.", pipeline);
-                createPipelineWithJsonInElasticsearch(client, pipeline, json);
-            }
-            else {
-                logger.debug("Pipeline [{}] already exists.", pipeline);
-            }
-        }
-
-        if (!isPipelineExist(client, pipeline)) {
-            logger.debug("Pipeline [{}] doesn't exist. Creating it.", pipeline);
-            createPipelineWithJsonInElasticsearch(client, pipeline, json);
+    public static void createPipeline(RestClient client, String root, String pipeline) throws Exception {
+        String json = getJsonContent(root, SettingsFinder.Defaults.PipelinesDir, pipeline);
+        if (json == null) {
+            logger.warn("Please rename [{}/{}/{}{}] to [{}/{}/{}{}].",
+                    root, SettingsFinder.Defaults.PipelineDir, pipeline, SettingsFinder.Defaults.JsonFileExtension,
+                    root, SettingsFinder.Defaults.PipelinesDir, pipeline, SettingsFinder.Defaults.JsonFileExtension);
+            json = getJsonContent(root, SettingsFinder.Defaults.PipelineDir, pipeline);
         }
+        createPipelineWithJsonInElasticsearch(client, pipeline, json);
     }
 
     /**
diff --git a/src/main/java/fr/pilato/elasticsearch/tools/util/ResourceList.java b/src/main/java/fr/pilato/elasticsearch/tools/util/ResourceList.java
index ff7fe3f..f12eb5d 100644
--- a/src/main/java/fr/pilato/elasticsearch/tools/util/ResourceList.java
+++ b/src/main/java/fr/pilato/elasticsearch/tools/util/ResourceList.java
@@ -184,6 +184,7 @@ public class ResourceList {
                         !key.equals(SettingsFinder.Defaults.ComponentTemplatesDir) &&
                         !key.equals(SettingsFinder.Defaults.TemplateDir) &&
                         !key.equals(SettingsFinder.Defaults.PipelineDir) &&
+                        !key.equals(SettingsFinder.Defaults.PipelinesDir) &&
                         !keys.contains(key)) {
                     logger.trace(" - found [{}].", key);
                     keys.add(key);
diff --git a/src/main/java/fr/pilato/elasticsearch/tools/util/SettingsFinder.java b/src/main/java/fr/pilato/elasticsearch/tools/util/SettingsFinder.java
index c4d9ecd..257609c 100644
--- a/src/main/java/fr/pilato/elasticsearch/tools/util/SettingsFinder.java
+++ b/src/main/java/fr/pilato/elasticsearch/tools/util/SettingsFinder.java
@@ -34,6 +34,8 @@ public class SettingsFinder {
 		public static String TemplateDir = "_template";
 		public static String IndexTemplatesDir = "_index_templates";
 		public static String ComponentTemplatesDir = "_component_templates";
+		public static String PipelinesDir = "_pipelines";
+		@Deprecated
 		public static String PipelineDir = "_pipeline";
 
 		/**


Test patch:
diff --git a/src/test/java/fr/pilato/elasticsearch/tools/AbstractBeyonderTest.java b/src/test/java/fr/pilato/elasticsearch/tools/AbstractBeyonderTest.java
index d180caa..6b88498 100644
--- a/src/test/java/fr/pilato/elasticsearch/tools/AbstractBeyonderTest.java
+++ b/src/test/java/fr/pilato/elasticsearch/tools/AbstractBeyonderTest.java
@@ -36,7 +36,6 @@ import java.io.IOException;
 import java.net.ConnectException;
 import java.util.List;
 import java.util.Map;
-import java.util.function.Supplier;
 
 import static java.util.Collections.singletonList;
 import static org.hamcrest.Matchers.containsString;
@@ -59,7 +58,8 @@ public abstract class AbstractBeyonderTest {
                                          List<String> indices,
                                          List<String> templates,
                                          List<String> componentTemplates,
-                                         List<String> indexTemplates) throws Exception;
+                                         List<String> indexTemplates,
+                                         List<String> pipelines) throws Exception;
 
     private static RestClient client;
 
@@ -123,7 +123,7 @@ public abstract class AbstractBeyonderTest {
         // Default dir es
         testBeyonder(null,
                 singletonList("twitter"),
-                null, null, null);
+                null, null, null, null);
     }
 
     @Test
@@ -131,7 +131,7 @@ public abstract class AbstractBeyonderTest {
         // Single index/single type
         testBeyonder("models/oneindexonetype",
                 singletonList("twitter"),
-                null, null, null);
+                null, null, null, null);
     }
 
     @Test
@@ -139,7 +139,7 @@ public abstract class AbstractBeyonderTest {
         // Custom settings (analyzer)
         testBeyonder("models/settingsanalyzer",
                 singletonList("twitter"),
-                null, null, null);
+                null, null, null, null);
     }
 
     @Test
@@ -147,7 +147,7 @@ public abstract class AbstractBeyonderTest {
         // 1 index and no type
         testBeyonder("models/oneindexnotype",
                 singletonList("twitter"),
-                null, null, null);
+                null, null, null, null);
     }
 
     @Test
@@ -155,12 +155,12 @@ public abstract class AbstractBeyonderTest {
         // 1 template
         testBeyonder("models/template",
                 null,
-                singletonList("twitter_template"), null, null);
+                singletonList("twitter_template"), null, null, null);
     }
 
     @Test
     public void testWrongClasspathDir() throws Exception {
         testBeyonder("models/bad-classpath-7/doesnotexist",
-                null, null, null, null);
+                null, null, null, null, null);
     }
 }
diff --git a/src/test/java/fr/pilato/elasticsearch/tools/BeyonderRestIT.java b/src/test/java/fr/pilato/elasticsearch/tools/BeyonderRestIT.java
index 65a4da5..a21278a 100644
--- a/src/test/java/fr/pilato/elasticsearch/tools/BeyonderRestIT.java
+++ b/src/test/java/fr/pilato/elasticsearch/tools/BeyonderRestIT.java
@@ -46,6 +46,7 @@ import static fr.pilato.elasticsearch.tools.JsonUtil.asMap;
 import static fr.pilato.elasticsearch.tools.updaters.ElasticsearchComponentTemplateUpdater.isComponentTemplateExist;
 import static fr.pilato.elasticsearch.tools.updaters.ElasticsearchIndexTemplateUpdater.isIndexTemplateExist;
 import static fr.pilato.elasticsearch.tools.updaters.ElasticsearchIndexUpdater.isIndexExist;
+import static fr.pilato.elasticsearch.tools.updaters.ElasticsearchPipelineUpdater.isPipelineExist;
 import static fr.pilato.elasticsearch.tools.updaters.ElasticsearchTemplateUpdater.isTemplateExist;
 import static java.util.Arrays.asList;
 import static java.util.Collections.singletonList;
@@ -112,7 +113,8 @@ public class BeyonderRestIT extends AbstractBeyonderTest {
                                 List<String> indices,
                                 List<String> templates,
                                 List<String> componentTemplates,
-                                List<String> indexTemplates) throws Exception {
+                                List<String> indexTemplates,
+                                List<String> pipelines) throws Exception {
         logger.info("--> scanning: [{}]", root);
         ElasticsearchBeyonder.start(client, root);
 
@@ -163,6 +165,18 @@ public class BeyonderRestIT extends AbstractBeyonderTest {
             }
             assertThat(allExists, is(true));
         }
+
+        // We can check if we have created the pipelines
+        if (pipelines != null) {
+            boolean allExists = true;
+
+            for (String pipeline : pipelines) {
+                if (!isPipelineExist(client, pipeline)) {
+                    allExists = false;
+                }
+            }
+            assertThat(allExists, is(true));
+        }
     }
 
     // This is a manual test as we don't have a real support of this in Beyonder yet
@@ -196,7 +210,12 @@ public class BeyonderRestIT extends AbstractBeyonderTest {
     @Test
 	public void testPipeline() throws Exception {
 		ElasticsearchBeyonder.start(client, "models/pipeline");
-		assertThat(ElasticsearchPipelineUpdater.isPipelineExist(client, "twitter_pipeline"), is(true));
+		assertThat(isPipelineExist(client, "twitter_pipeline"), is(true));
+	}
+
+    @Test
+	public void testPipelines() throws Exception {
+        testBeyonder("models/pipelines", null, null, null, null, singletonList("twitter_pipeline"));
 	}
 
     @Test
@@ -205,7 +224,7 @@ public class BeyonderRestIT extends AbstractBeyonderTest {
         {
             testBeyonder("models/update-settings/step1",
                     singletonList("twitter"),
-                    null, null, null);
+                    null, null, null, null);
             Map<String, Object> oldSettings = asMap(client.performRequest(new Request("GET", "/twitter/_settings")));
             String numberOfReplicas = BeanUtils.getProperty(oldSettings, "twitter.settings.index.number_of_replicas");
             assertThat(numberOfReplicas, equalTo("0"));
@@ -215,7 +234,7 @@ public class BeyonderRestIT extends AbstractBeyonderTest {
         {
             testBeyonder("models/update-settings/step2",
                     singletonList("twitter"),
-                    null, null, null);
+                    null, null, null, null);
             Map<String, Object> settings = asMap(client.performRequest(new Request("GET", "/twitter/_settings")));
             String numberOfReplicas = BeanUtils.getProperty(settings, "twitter.settings.index.number_of_replicas");
             assertThat(numberOfReplicas, equalTo("1"));
@@ -228,7 +247,7 @@ public class BeyonderRestIT extends AbstractBeyonderTest {
         {
             testBeyonder("models/update-mapping/step1",
                     singletonList("twitter"),
-                    null, null, null);
+                    null, null, null, null);
 
             Map<String, Object> mapping = asMap(client.performRequest(new Request("GET", "/twitter/_mappings")));
             String bar = BeanUtils.getProperty(mapping, "twitter.mappings.properties.bar");
@@ -243,7 +262,7 @@ public class BeyonderRestIT extends AbstractBeyonderTest {
         {
             testBeyonder("models/update-mapping/step2",
                     singletonList("twitter"),
-                    null, null, null);
+                    null, null, null, null);
 
             Map<String, Object> mapping = asMap(client.performRequest(new Request("GET", "/twitter/_mappings")));
             String bar = BeanUtils.getProperty(mapping, "twitter.mappings.properties.bar");
@@ -262,7 +281,7 @@ public class BeyonderRestIT extends AbstractBeyonderTest {
                 null,
                 null,
                 asList("component1", "component2"),
-                singletonList("template_1"));
+                singletonList("template_1"), null);
     }
 
 
diff --git a/src/test/java/fr/pilato/elasticsearch/tools/BeyonderTransportIT.java b/src/test/java/fr/pilato/elasticsearch/tools/BeyonderTransportIT.java
index 22241f9..db92b81 100644
--- a/src/test/java/fr/pilato/elasticsearch/tools/BeyonderTransportIT.java
+++ b/src/test/java/fr/pilato/elasticsearch/tools/BeyonderTransportIT.java
@@ -20,15 +20,12 @@
 package fr.pilato.elasticsearch.tools;
 
 import org.apache.commons.beanutils.BeanUtils;
-import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
-import org.elasticsearch.action.admin.indices.template.delete.DeleteComponentTemplateAction;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.client.transport.NoNodeAvailableException;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
-import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.transport.client.PreBuiltTransportClient;
 import org.junit.After;
 import org.junit.AfterClass;
@@ -42,11 +39,8 @@ import java.io.IOException;
 import java.net.InetSocketAddress;
 import java.util.List;
 import java.util.Map;
-import java.util.function.Function;
-import java.util.function.Supplier;
 
 import static fr.pilato.elasticsearch.tools.updaters.ElasticsearchIndexUpdater.isIndexExist;
-import static fr.pilato.elasticsearch.tools.updaters.ElasticsearchIndexUpdater.removeIndexInElasticsearch;
 import static fr.pilato.elasticsearch.tools.updaters.ElasticsearchTemplateUpdater.isTemplateExist;
 import static java.util.Collections.singletonList;
 import static org.hamcrest.MatcherAssert.assertThat;
@@ -117,7 +111,7 @@ public class BeyonderTransportIT extends AbstractBeyonderTest {
         {
             testBeyonder("models/update-settings/step1",
                     singletonList("twitter"),
-                    null, null, null);
+                    null, null, null, null);
             GetSettingsResponse settings = client.admin().indices().prepareGetSettings("twitter").get();
             String numberOfReplicas = settings.getSetting("twitter", "index.number_of_replicas");
             assertThat(numberOfReplicas, equalTo("0"));
@@ -127,7 +121,7 @@ public class BeyonderTransportIT extends AbstractBeyonderTest {
         {
             testBeyonder("models/update-settings/step2",
                     singletonList("twitter"),
-                    null, null, null);
+                    null, null, null, null);
             GetSettingsResponse settings = client.admin().indices().prepareGetSettings("twitter").get();
             String numberOfReplicas = settings.getSetting("twitter", "index.number_of_replicas");
             assertThat(numberOfReplicas, equalTo("1"));
@@ -140,7 +134,7 @@ public class BeyonderTransportIT extends AbstractBeyonderTest {
         {
             testBeyonder("models/update-mapping/step1",
                     singletonList("twitter"),
-                    null, null, null);
+                    null, null, null, null);
             Map<String, Object> properties = client.admin().indices().prepareGetMappings("twitter").get().getMappings().get("twitter").get("_doc").getSourceAsMap();
             String bar = BeanUtils.getProperty(properties, "properties.bar");
             String foo = BeanUtils.getProperty(properties, "properties.foo");
@@ -154,7 +148,7 @@ public class BeyonderTransportIT extends AbstractBeyonderTest {
         {
             testBeyonder("models/update-mapping/step2",
                     singletonList("twitter"),
-                    null, null, null);
+                    null, null, null, null);
 
             Map<String, Object> properties = client.admin().indices().prepareGetMappings("twitter").get().getMappings().get("twitter").get("_doc").getSourceAsMap();
             String bar = BeanUtils.getProperty(properties, "properties.bar");
@@ -170,7 +164,8 @@ public class BeyonderTransportIT extends AbstractBeyonderTest {
                                 List<String> indices,
                                 List<String> templates,
                                 List<String> componentTemplates,
-                                List<String> indexTemplates) throws Exception {
+                                List<String> indexTemplates,
+                                List<String> pipelines) throws Exception {
         String newRoot = "transport/" + root;
         logger.info("--> scanning: [{}]", newRoot);
         ElasticsearchBeyonder.start(client, newRoot);
diff --git a/src/test/java/fr/pilato/elasticsearch/tools/BeyonderUnitTest.java b/src/test/java/fr/pilato/elasticsearch/tools/BeyonderUnitTest.java
index 4997e25..013c7cf 100644
--- a/src/test/java/fr/pilato/elasticsearch/tools/BeyonderUnitTest.java
+++ b/src/test/java/fr/pilato/elasticsearch/tools/BeyonderUnitTest.java
@@ -45,7 +45,8 @@ public class BeyonderUnitTest extends AbstractBeyonderTest {
                                 List<String> indices,
                                 List<String> templates,
                                 List<String> componentTemplates,
-                                List<String> indexTemplates) throws IOException, URISyntaxException {
+                                List<String> indexTemplates,
+                                List<String> pipelines) throws IOException, URISyntaxException {
         logger.info("--> scanning: [{}]", root);
 
         List<String> indexNames = ResourceList.findIndexNames(root);
@@ -88,6 +89,13 @@ public class BeyonderUnitTest extends AbstractBeyonderTest {
                 throw new RuntimeException("Our test is failing...");
             }
         });
+        check(ResourceList.getResourceNames(root, SettingsFinder.Defaults.PipelinesDir), pipelines, (name) -> {
+            try {
+                return getJsonContent(root, SettingsFinder.Defaults.PipelinesDir, name);
+            } catch (IOException e) {
+                throw new RuntimeException("Our test is failing...");
+            }
+        });
     }
 
     private void check(List<String> names, List<String> expectedNames,
@@ -133,12 +141,12 @@ public class BeyonderUnitTest extends AbstractBeyonderTest {
         // 1 _settings
         testBeyonder("models/update-mapping/step1",
                 singletonList("twitter"),
-                null, null, null);
+                null, null, null, null);
 
         // 2 _update_mapping
         testBeyonder("models/update-mapping/step2",
                 singletonList("twitter"),
-                null, null, null);
+                null, null, null, null);
     }
 
     @Test
@@ -146,12 +154,12 @@ public class BeyonderUnitTest extends AbstractBeyonderTest {
         // 1 _settings
         testBeyonder("models/update-settings/step1",
                 singletonList("twitter"),
-                null, null, null);
+                null, null, null, null);
 
         // 2 _update_settings
         testBeyonder("models/update-settings/step2",
                 singletonList("twitter"),
-                null, null, null);
+                null, null, null, null);
     }
 
     @Test
@@ -161,6 +169,18 @@ public class BeyonderUnitTest extends AbstractBeyonderTest {
                 null,
                 null,
                 asList("component1", "component2"),
-                singletonList("template_1"));
+                singletonList("template_1"),
+                null);
+    }
+
+    @Test
+    public void testPipelines() throws Exception {
+        // 1 template
+        testBeyonder("models/pipelines",
+                null,
+                null,
+                null,
+                null,
+                singletonList("twitter_pipeline"));
     }
 }
diff --git a/src/test/resources/models/pipelines/_pipelines/twitter_pipeline.json b/src/test/resources/models/pipelines/_pipelines/twitter_pipeline.json
new file mode 100644
index 0000000..60b74ea
--- /dev/null
+++ b/src/test/resources/models/pipelines/_pipelines/twitter_pipeline.json
@@ -0,0 +1,11 @@
+{
+  "description" : "Twitter pipeline",
+  "processors" : [
+    {
+      "set" : {
+        "field": "foo",
+        "value": "bar"
+      }
+    }
+  ]
+}
\ No newline at end of file