Repository: exasol/integration-test-docker-environment (language: python)

Issue:
test environment for external db not working
# How to reproduce

1. Start `SpawnTestEnvironment` with type `external`. 
2. test environment crashes

## RCA

1. IsDatabaseThread uses db container to check database status via `exaplus`
2. The db container is not available when starting environment for external db

## Solution

1. Skip tests for startup when starting test environment for external db
2. Add integration test for test environment with external db


test environment for external db not working
# How to reproduce

1. Start `SpawnTestEnvironment` with type `external`. 
2. test environment crashes

## RCA

1. IsDatabaseThread uses db container to check database status via `exaplus`
2. The db container is not available when starting environment for external db

## Solution

1. Skip tests for startup when starting test environment for external db
2. Add integration test for test environment with external db




Discussion / hints:

Fix patch:
diff --git a/doc/changes/changelog.md b/doc/changes/changelog.md
index dc521e727..e8379d560 100644
--- a/doc/changes/changelog.md
+++ b/doc/changes/changelog.md
@@ -1,5 +1,6 @@
 # Changes
 
+* [1.1.0](changes_1.1.0.md)
 * [1.0.0](changes_1.0.0.md)
 * [0.11.0](changes_0.11.0.md)
 * [0.10.0](changes_0.10.0.md)
@@ -20,6 +21,7 @@
 ---
 hidden:
 ---
+changes_1.1.0
 changes_1.0.0
 changes_0.11.0
 changes_0.10.0
diff --git a/doc/changes/changes_1.1.0.md b/doc/changes/changes_1.1.0.md
new file mode 100644
index 000000000..32d37b817
--- /dev/null
+++ b/doc/changes/changes_1.1.0.md
@@ -0,0 +1,36 @@
+# Integration-Test-Docker-Environment 1.1.0, released t.b.d.
+
+Code name: Bugfix for test environment with external db
+
+## Summary
+This release fixes a bug when starting the test environment with external db.
+
+### Supported Exasol Versions
+
+* **7.0**: up to 7.0.20, **except 7.0.5**
+* **7.1**: up to 7.1.14
+
+If you need further versions, please open an issue.
+
+### Tested Docker Runtimes
+
+- Docker Default Runtime
+
+## Bug Fixes:
+- #263: Fixed test environment with external db
+
+## Features / Enhancements:
+
+n/a
+
+## Refactoring:
+
+n/a
+
+## Documentation:
+
+n/a
+
+## Security:
+
+n/a
diff --git a/poetry.lock b/poetry.lock
index 1118de4ad..3ff42ee6b 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -166,7 +166,7 @@ Sphinx = "^4.5"
 type = "git"
 url = "https://github.com/exasol/sphinx-github-pages-generator.git"
 reference = "HEAD"
-resolved_reference = "8140484ecb999f5af023b84f2ade3db2ed9722ce"
+resolved_reference = "16143e6acac138875730eb4a478cbca663bf6b48"
 
 [[package]]
 name = "filelock"
diff --git a/pyproject.toml b/pyproject.toml
index 966c41613..6d467fb61 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "exasol-integration-test-docker-environment"
-version = "1.0.0"
+version = "1.1.0"
 description = "Integration Test Docker Environment for Exasol"
 
 license = "MIT"
diff --git a/setup.py b/setup.py
index dca956dbb..84660b095 100644
--- a/setup.py
+++ b/setup.py
@@ -93,7 +93,7 @@
 
 setup_kwargs = {
     'name': 'exasol-integration-test-docker-environment',
-    'version': '1.0.0',
+    'version': '1.1.0',
     'description': 'Integration Test Docker Environment for Exasol',
     'long_description': 'Integration Test Docker Environment\n===================================\n\nThis project provides a command line interface and a Python API layer to\nstart a test environment with an `Exasol\nDocker-DB <https://hub.docker.com/r/exasol/docker-db>`_. Both start an\nExasol Docker-DB container, but the API Layer has extended functionality\nand also can start an associated test container for whose content the\nclient is responsible.\n\nDocumentation\n-------------\n\n`Documentation for the current main branch <https://exasol.github.io/integration-test-docker-environment/main>`_ is hosted on the Github Pages of this project.\n`Here <https://exasol.github.io/integration-test-docker-environment>`_  is a list of documentations for previous releases.\n',
     'author': 'Torsten Kilias',


Test patch:
diff --git a/exasol_integration_test_docker_environment/lib/test_environment/database_waiters/wait_for_external_database.py b/exasol_integration_test_docker_environment/lib/test_environment/database_waiters/wait_for_external_database.py
index d9d9625b5..e15b05b0f 100644
--- a/exasol_integration_test_docker_environment/lib/test_environment/database_waiters/wait_for_external_database.py
+++ b/exasol_integration_test_docker_environment/lib/test_environment/database_waiters/wait_for_external_database.py
@@ -1,16 +1,9 @@
-import time
-from datetime import datetime, timedelta
-
 import luigi
-from docker.models.containers import Container
 
 from exasol_integration_test_docker_environment.lib.base.docker_base_task import DockerBaseTask
 from exasol_integration_test_docker_environment.lib.base.json_pickle_parameter import JsonPickleParameter
-from exasol_integration_test_docker_environment.lib.data.container_info import ContainerInfo
 from exasol_integration_test_docker_environment.lib.data.database_credentials import DatabaseCredentialsParameter
 from exasol_integration_test_docker_environment.lib.data.database_info import DatabaseInfo
-from exasol_integration_test_docker_environment.lib.test_environment.database_waiters.is_database_ready_thread import \
-    IsDatabaseReadyThread
 
 
 class WaitForTestExternalDatabase(DockerBaseTask,
@@ -21,58 +14,7 @@ class WaitForTestExternalDatabase(DockerBaseTask,
     attempt = luigi.IntParameter(1)
 
     def run_task(self):
-        with self._get_docker_client() as docker_client:
-            db_container_name = self.database_info.container_info.container_name
-            is_database_ready = self.wait_for_database_startup(db_container_name)
-            self.return_object(is_database_ready)
-
-    def wait_for_database_startup(self, db_container: Container):
-        is_database_ready_thread = self.start_wait_threads(db_container)
-        is_database_ready = self.wait_for_threads(is_database_ready_thread)
-        self.join_threads(is_database_ready_thread)
-        return is_database_ready
-
-    def start_wait_threads(self, db_container):
-        is_database_ready_thread = IsDatabaseReadyThread(self.logger,
-                                                         self.database_info,
-                                                         self.get_database_credentials(),
-                                                         db_container)
-        is_database_ready_thread.start()
-        return is_database_ready_thread
-
-    def join_threads(self, is_database_ready_thread: IsDatabaseReadyThread):
-        is_database_ready_thread.stop()
-        is_database_ready_thread.join()
-
-    def wait_for_threads(self, is_database_ready_thread: IsDatabaseReadyThread):
-        is_database_ready = False
-        reason = None
-        start_time = datetime.now()
-        while (True):
-            if is_database_ready_thread.finish:
-                is_database_ready = True
-                break
-            if self.timeout_occured(start_time):
-                reason = f"timeout after after {self.db_startup_timeout_in_seconds} seconds"
-                is_database_ready = False
-                break
-            time.sleep(1)
-        if not is_database_ready:
-            self.log_database_not_ready(is_database_ready_thread, reason)
-        is_database_ready_thread.stop()
-        return is_database_ready
-
-    def log_database_not_ready(self, is_database_ready_thread, reason):
-        log_information = f"""
-========== IsDatabaseReadyThread output db connection: ============
-{is_database_ready_thread.output_db_connection}
-========== IsDatabaseReadyThread output bucketfs connection: ============
-{is_database_ready_thread.output_bucketfs_connection}
-"""
-        self.logger.warning(
-            'Database startup failed for following reason "%s", here some debug information \n%s',
-            reason, log_information)
-
-    def timeout_occured(self, start_time):
-        timeout = timedelta(seconds=self.db_startup_timeout_in_seconds)
-        return datetime.now() - start_time > timeout
+        # Since we can't assume that the test container exists, we cannot connect easily here
+        # to the external database (correct way would be by using an SQL client).
+        # For now, we simply assume that the external database is already ready and return just True.
+        self.return_object(True)
diff --git a/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_environment_with_external_db.py b/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_environment_with_external_db.py
index f09e6d6ed..b4ad5a261 100644
--- a/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_environment_with_external_db.py
+++ b/exasol_integration_test_docker_environment/lib/test_environment/spawn_test_environment_with_external_db.py
@@ -30,7 +30,9 @@ def create_network_task(self, attempt: int):
                 reuse=self.reuse_test_container,
                 attempt=attempt,
                 test_container_name=self.test_container_name,
-                network_name=self.network_name
+                network_name=self.network_name,
+                no_cleanup_after_success=self.reuse_test_container,
+                no_cleanup_after_failure=self.reuse_test_container
             )
 
     def create_spawn_database_task(self, network_info: DockerNetworkInfo,
diff --git a/exasol_integration_test_docker_environment/test/test_api_external_test_environment.py b/exasol_integration_test_docker_environment/test/test_api_external_test_environment.py
new file mode 100644
index 000000000..376970eff
--- /dev/null
+++ b/exasol_integration_test_docker_environment/test/test_api_external_test_environment.py
@@ -0,0 +1,82 @@
+import unittest
+from sys import stderr
+
+from exasol_integration_test_docker_environment.lib.api.common import generate_root_task, run_task
+from exasol_integration_test_docker_environment.lib.data.environment_info import EnvironmentInfo
+from exasol_integration_test_docker_environment.lib.data.environment_type import EnvironmentType
+from exasol_integration_test_docker_environment.lib.docker import ContextDockerClient
+from exasol_integration_test_docker_environment.lib.docker.container.utils import remove_docker_container
+from exasol_integration_test_docker_environment.lib.test_environment.spawn_test_environment import SpawnTestEnvironment
+from exasol_integration_test_docker_environment.test.get_test_container_content import get_test_container_content
+from exasol_integration_test_docker_environment.testing import utils
+from exasol_integration_test_docker_environment.testing.api_test_environment import ApiTestEnvironment
+
+
+class APISpawnTestExternalEnvironmentTest(unittest.TestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        print(f"SetUp {cls.__name__}", file=stderr)
+        cls.test_environment = ApiTestEnvironment(cls)
+        cls.docker_environment_name = cls.__name__
+        cls.environment = \
+            cls.test_environment.spawn_docker_test_environment(
+                name=cls.docker_environment_name)
+        cls.ext_environment_name = "APISpawnExternalTestExternalEnvironmentTest"
+
+        task_creator = \
+            lambda: generate_root_task(task_class=SpawnTestEnvironment,
+                                       environment_type=EnvironmentType.external_db,
+                                       environment_name=cls.ext_environment_name,
+                                       external_exasol_db_host=cls.environment.database_host,
+                                       external_exasol_db_port=cls.environment.database_port,
+                                       external_exasol_bucketfs_port=cls.environment.bucketfs_port,
+                                       external_exasol_db_user=cls.environment.db_username,
+                                       external_exasol_db_password=cls.environment.db_password,
+                                       external_exasol_bucketfs_write_password=cls.environment.bucketfs_password,
+                                       external_exasol_xmlrpc_host=None,
+                                       external_exasol_xmlrpc_port=443,
+                                       external_exasol_xmlrpc_user="admin",
+                                       external_exasol_xmlrpc_password=None,
+                                       external_exasol_xmlrpc_cluster_name="cluster1",
+                                       no_test_container_cleanup_after_success=True,
+                                       no_test_container_cleanup_after_failure=False,
+                                       reuse_test_container=True,
+                                       test_container_content=get_test_container_content())
+        cls.ext_environment_info: EnvironmentInfo = run_task(task_creator, 1, None)
+
+    @classmethod
+    def tearDownClass(cls):
+        utils.close_environments(cls.environment, cls.test_environment)
+        with ContextDockerClient() as docker_client:
+            containers = [c.name for c in docker_client.containers.list() if cls.ext_environment_name in c.name]
+            remove_docker_container(containers)
+
+    def test_external_db(self):
+        with ContextDockerClient() as docker_client:
+            containers = [c.name for c in docker_client.containers.list() if self.docker_environment_name in c.name]
+            self.assertEqual(len(containers), 1,
+                             f"Not exactly 1 containers in {containers}.")
+            db_container = [c for c in containers if "db_container" in c]
+            self.assertEqual(len(db_container), 1,
+                             f"Found no db container in {containers}.")
+            containers = [c.name for c in docker_client.containers.list() if self.ext_environment_name in c.name]
+            self.assertEqual(len(containers), 1,
+                             f"Not exactly 1 containers in {containers}.")
+            test_container = [c for c in containers if "test_container" in c]
+            self.assertEqual(len(test_container), 1,
+                             f"Found no test container in {containers}.")
+
+    def test_docker_available_in_test_container(self):
+        environment_info = self.ext_environment_info
+        with ContextDockerClient() as docker_client:
+            test_container = docker_client.containers.get(environment_info.test_container_info.container_name)
+            exit_result = test_container.exec_run("docker ps")
+            exit_code = exit_result[0]
+            output = exit_result[1]
+            self.assertEqual(exit_code, 0,
+                             f"Error while executing 'docker ps' in test container got output\n {output}.")
+
+
+if __name__ == '__main__':
+    unittest.main()