Repository: dadoonet/elasticsearch-beyonder
Pull request #331 (2025-02-28 16:00:01)

Switch to TestContainers for IT


Patch:
diff --git a/pom.xml b/pom.xml
index 5bdf0e4..771cae4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -27,11 +27,6 @@ Elasticsearch beyonder provides automatic index, mapping, alias, template creati
         <commons.io.version>2.18.0</commons.io.version>
         <jackson.version>2.15.3</jackson.version>
 
-        <!-- For integration tests using Docker or external cluster -->
-        <integ.elasticsearch.image>docker.elastic.co/elasticsearch/elasticsearch</integ.elasticsearch.image>
-        <integ.elasticsearch.version>${elasticsearch.version}</integ.elasticsearch.version>
-        <tests.cluster.rest.port>9200</tests.cluster.rest.port>
-
         <skipTests>false</skipTests>
         <skipUnitTests>${skipTests}</skipUnitTests>
         <skipIntegTests>${skipTests}</skipIntegTests>
@@ -149,6 +144,13 @@ Elasticsearch beyonder provides automatic index, mapping, alias, template creati
             <version>4.13.2</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.testcontainers</groupId>
+            <artifactId>elasticsearch</artifactId>
+            <version>1.20.4</version>
+            <scope>test</scope>
+        </dependency>
+
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-core</artifactId>
@@ -303,59 +305,6 @@ Elasticsearch beyonder provides automatic index, mapping, alias, template creati
                     </execution>
                 </executions>
             </plugin>
-            <!-- For IT using Docker -->
-            <plugin>
-                <groupId>io.fabric8</groupId>
-                <artifactId>docker-maven-plugin</artifactId>
-                <version>0.43.4</version>
-                <configuration>
-                    <skip>${skipIntegTests}</skip>
-                    <images>
-                        <image>
-                            <alias>elasticsearch-beyonder</alias>
-                            <name>dadoonet/elasticsearch-beyonder:${project.version}</name>
-                            <build>
-                                <from>${integ.elasticsearch.image}:${integ.elasticsearch.version}</from>
-                                <env>
-                                    <discovery.type>single-node</discovery.type>
-                                    <ELASTIC_PASSWORD>changeme</ELASTIC_PASSWORD>
-                                </env>
-                            </build>
-                            <run>
-                                <ports>
-                                    <port>tests.cluster.rest.port:9200</port>
-                                </ports>
-                                <wait>
-                                    <http>
-                                        <url>https://localhost:${tests.cluster.rest.port}/</url>
-                                        <status>200..499</status>
-                                        <allowAllHosts>true</allowAllHosts>
-                                    </http>
-                                    <time>60000</time>
-                                </wait>
-                            </run>
-                        </image>
-                    </images>
-                </configuration>
-                <executions>
-                    <execution>
-                        <id>start-elasticsearch</id>
-                        <phase>pre-integration-test</phase>
-                        <goals>
-                            <goal>build</goal>
-                            <goal>stop</goal>
-                            <goal>start</goal>
-                        </goals>
-                    </execution>
-                    <execution>
-                        <id>stop-elasticsearch</id>
-                        <phase>post-integration-test</phase>
-                        <goals>
-                            <goal>stop</goal>
-                        </goals>
-                    </execution>
-                </executions>
-            </plugin>
         </plugins>
     </build>
 
@@ -377,7 +326,7 @@ Elasticsearch beyonder provides automatic index, mapping, alias, template creati
                     <plugin>
                         <groupId>org.apache.maven.plugins</groupId>
                         <artifactId>maven-javadoc-plugin</artifactId>
-                        <version>3.6.2</version>
+                        <version>3.11.2</version>
                         <executions>
                             <execution>
                                 <id>attach-javadocs</id>


Test patch:
diff --git a/src/test/java/fr/pilato/elasticsearch/tools/AbstractBeyonderTest.java b/src/test/java/fr/pilato/elasticsearch/tools/AbstractBeyonderTest.java
index c7c4e5e..c42b1b2 100644
--- a/src/test/java/fr/pilato/elasticsearch/tools/AbstractBeyonderTest.java
+++ b/src/test/java/fr/pilato/elasticsearch/tools/AbstractBeyonderTest.java
@@ -28,12 +28,18 @@ import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.client.RestClientBuilder;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
 import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.net.ConnectException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.List;
 import java.util.Map;
 
@@ -70,38 +76,69 @@ public abstract class AbstractBeyonderTest {
         return client;
     }
 
+    @ClassRule
+    public static final TemporaryFolder folder = new TemporaryFolder();
+    protected static Path rootTmpDir;
+
+    @BeforeClass
+    public static void createTmpDir() throws IOException {
+        folder.create();
+        rootTmpDir = Paths.get(folder.getRoot().toURI());
+    }
+
     private static void startRestClient() throws IOException {
         if (client == null) {
-            RestClientBuilder builder = RestClient.builder(HttpHost.create(testCluster));
-            final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
-            credentialsProvider.setCredentials(AuthScope.ANY,
-                    new UsernamePasswordCredentials(testClusterUser, testClusterPass));
-
-            builder.setHttpClientConfigCallback(hcb -> hcb
-                    .setDefaultCredentialsProvider(credentialsProvider)
-                    .setSSLContext(SSLUtils.yesSSLContext())
-            );
-
-            client = builder.build();
-            testClusterRunning();
+            client = buildClient(testCluster, testClusterUser, null);
+            try {
+                if (!testClusterRunning()) {
+                    TestContainerHelper containerHelper = new TestContainerHelper();
+                    String url = containerHelper.startElasticsearch();
+                    Path clusterCaCrtPath = null;
+                    if (containerHelper.getCertAsBytes() != null) {
+                        clusterCaCrtPath = rootTmpDir.resolve("cluster-ca.crt");
+                        Files.write(clusterCaCrtPath, containerHelper.getCertAsBytes());
+                    }
+                    client = buildClient(url, DEFAULT_TEST_USER, clusterCaCrtPath);
+                }
+            } catch (ConnectException e) {
+                // If we have an exception here, let's ignore the test
+                logger.warn("Integration tests are skipped: [{}]", e.getMessage());
+                assumeThat("Integration tests are skipped", e.getMessage(), not(containsString("Connection refused")));
+            } catch (IOException e) {
+                logger.error("Full error is", e);
+                throw e;
+            }
         }
     }
 
-    private static void testClusterRunning() throws IOException {
+    private static boolean testClusterRunning() throws IOException {
         try {
             Response response = client.performRequest(new Request("GET", "/"));
             Map<String, Object> asMap = (Map<String, Object>) JsonUtil.asMap(response).get("version");
             logger.info("Starting integration tests against an external cluster running elasticsearch [{}]", asMap.get("number"));
+            return true;
         } catch (ConnectException e) {
-            // If we have an exception here, let's ignore the test
-            logger.warn("Integration tests are skipped: [{}]", e.getMessage());
-            assumeThat("Integration tests are skipped", e.getMessage(), not(containsString("Connection refused")));
-        } catch (IOException e) {
-            logger.error("Full error is", e);
-            throw e;
+            return false;
         }
     }
 
+    private static RestClient buildClient(String url, String user, Path caCertificate) {
+        RestClientBuilder builder = RestClient.builder(HttpHost.create(url));
+        final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
+        credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(user, testClusterPass));
+        builder.setHttpClientConfigCallback(hcb -> {
+                hcb.setDefaultCredentialsProvider(credentialsProvider);
+                if (caCertificate == null) {
+                    hcb.setSSLContext(SSLUtils.yesSSLContext());
+                } else {
+                    hcb.setSSLContext(SSLUtils.sslContextFromHttpCaCrt(caCertificate));
+                }
+                return hcb;
+            }
+        );
+        return builder.build();
+    }
+
     @FunctionalInterface
     public interface ThrowingConsumer<E extends Exception> {
         void run() throws E;
diff --git a/src/test/java/fr/pilato/elasticsearch/tools/SSLUtils.java b/src/test/java/fr/pilato/elasticsearch/tools/SSLUtils.java
index 86d6978..685866e 100644
--- a/src/test/java/fr/pilato/elasticsearch/tools/SSLUtils.java
+++ b/src/test/java/fr/pilato/elasticsearch/tools/SSLUtils.java
@@ -20,7 +20,19 @@
 package fr.pilato.elasticsearch.tools;
 
 import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManagerFactory;
 import javax.net.ssl.X509TrustManager;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.security.KeyManagementException;
+import java.security.KeyStore;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.security.cert.Certificate;
+import java.security.cert.CertificateException;
+import java.security.cert.CertificateFactory;
 import java.security.cert.X509Certificate;
 
 public class SSLUtils {
@@ -62,4 +74,25 @@ public class SSLUtils {
 
         return yesCtx;
     }
+
+    public static SSLContext sslContextFromHttpCaCrt(Path file) {
+        try(InputStream in = Files.newInputStream(file)) {
+            CertificateFactory cf = CertificateFactory.getInstance("X.509");
+            Certificate certificate = cf.generateCertificate(in);
+
+            final KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType());
+            keyStore.load(null, null);
+            keyStore.setCertificateEntry("elasticsearch-ca", certificate);
+
+            TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
+            tmf.init(keyStore);
+
+            SSLContext sslContext = SSLContext.getInstance("TLS");
+            sslContext.init(null, tmf.getTrustManagers(), null);
+            return sslContext;
+        } catch (CertificateException | NoSuchAlgorithmException | KeyManagementException | KeyStoreException |
+                 IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
 }
diff --git a/src/test/java/fr/pilato/elasticsearch/tools/TestContainerHelper.java b/src/test/java/fr/pilato/elasticsearch/tools/TestContainerHelper.java
new file mode 100644
index 0000000..41a1f83
--- /dev/null
+++ b/src/test/java/fr/pilato/elasticsearch/tools/TestContainerHelper.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to David Pilato (the "Author") under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Author licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package fr.pilato.elasticsearch.tools;
+
+import org.apache.commons.io.IOUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testcontainers.elasticsearch.ElasticsearchContainer;
+import org.testcontainers.utility.DockerImageName;
+
+import java.io.IOException;
+import java.util.Properties;
+
+/**
+ * This creates a TestContainer Elasticsearch instance
+ */
+class TestContainerHelper {
+
+    private static final Logger log = LoggerFactory.getLogger(TestContainerHelper.class);
+
+    ElasticsearchContainer elasticsearch;
+    private byte[] certAsBytes;
+
+    /**
+     * Start the container
+     *
+     * @throws IOException in case of error
+     */
+    String startElasticsearch() throws IOException {
+        Properties props = new Properties();
+        props.load(TestContainerHelper.class.getResourceAsStream("/beyonder-tests.properties"));
+        String version = props.getProperty("elasticsearch.version");
+
+        if (elasticsearch == null) {
+            // Start the container. This step might take some time...
+            log.info("Starting testcontainers with Elasticsearch [{}].", version);
+
+            elasticsearch = new ElasticsearchContainer(
+                    DockerImageName.parse("docker.elastic.co/elasticsearch/elasticsearch")
+                            .withTag(version))
+                    .withReuse(true)
+                    .withPassword(AbstractBeyonderTest.testClusterPass);
+            elasticsearch.start();
+
+            // Try to get the https certificate if exists
+            try {
+                certAsBytes = elasticsearch.copyFileFromContainer(
+                        "/usr/share/elasticsearch/config/certs/http_ca.crt",
+                        IOUtils::toByteArray);
+                log.debug("Found an https elasticsearch cert for version [{}].", version);
+            } catch (Exception e) {
+                log.warn("We did not find the https elasticsearch cert for version [{}].", version);
+            }
+        } else {
+            log.info("Testcontainers with Elasticsearch [{}] was previously started", version);
+        }
+
+        String url = "https://" + elasticsearch.getHttpHostAddress();
+
+        log.info("Elasticsearch container is now running at {}", url);
+        return url;
+    }
+
+    public byte[] getCertAsBytes() {
+        return certAsBytes;
+    }
+}
diff --git a/src/test/resources/beyonder-tests.properties b/src/test/resources/beyonder-tests.properties
new file mode 100644
index 0000000..000dadf
--- /dev/null
+++ b/src/test/resources/beyonder-tests.properties
@@ -0,0 +1,1 @@
+elasticsearch.version=${elasticsearch.version}