Repository: exasol/integration-test-docker-environment (language: python)

Issue:
Fix exception if goals are not available for docker builds
## Error:

```
2021-03-24T07:12:49.7887300Z ##[group]Run vagrant ssh -c "cd /script-languages-release; ./exaslct build --flavor-path flavors/standard-EXASOL-7.0.0/ --goal udf_client_deps"

2021-03-24T07:12:49.7888810Z [36;1mvagrant ssh -c "cd /script-languages-release; ./exaslct build --flavor-path flavors/standard-EXASOL-7.0.0/ --goal udf_client_deps"[0m

2021-03-24T07:14:09.1595850Z Traceback (most recent call last):
2021-03-24T07:14:09.1597480Z   File "/script-languages-release/exaslct_src/exaslct.py", line 25, in <module>
2021-03-24T07:14:09.1598070Z     cli()
2021-03-24T07:14:09.1599360Z   File "/home/vagrant/.local/share/virtualenvs/script-languages-release-igq1tTpT/lib/python3.6/site-packages/click/core.py", line 829, in __call__
2021-03-24T07:14:09.1601130Z     return self.main(*args, **kwargs)
2021-03-24T07:14:09.1602540Z   File "/home/vagrant/.local/share/virtualenvs/script-languages-release-igq1tTpT/lib/python3.6/site-packages/click/core.py", line 782, in main
2021-03-24T07:14:09.1603480Z     rv = self.invoke(ctx)
2021-03-24T07:14:09.1604910Z   File "/home/vagrant/.local/share/virtualenvs/script-languages-release-igq1tTpT/lib/python3.6/site-packages/click/core.py", line 1259, in invoke
2021-03-24T07:14:09.1611870Z     return _process_result(sub_ctx.command.invoke(sub_ctx))
2021-03-24T07:14:09.1613640Z   File "/home/vagrant/.local/share/virtualenvs/script-languages-release-igq1tTpT/lib/python3.6/site-packages/click/core.py", line 1066, in invoke
2021-03-24T07:14:09.1614710Z     return ctx.invoke(self.callback, **ctx.params)
2021-03-24T07:14:09.1616290Z   File "/home/vagrant/.local/share/virtualenvs/script-languages-release-igq1tTpT/lib/python3.6/site-packages/click/core.py", line 610, in invoke
2021-03-24T07:14:09.1617250Z     return callback(*args, **kwargs)
2021-03-24T07:14:09.1618310Z   File "/script-languages-release/exaslct_src/exaslct/cli/commands/build.py", line 74, in build
2021-03-24T07:14:09.1619160Z     success, task = run_task(task_creator, workers, task_dependencies_dot_file)
2021-03-24T07:14:09.1621220Z   File "/home/vagrant/.local/share/virtualenvs/script-languages-release-igq1tTpT/lib/python3.6/site-packages/exasol_integration_test_docker_environment/cli/common.py", line 93, in run_task
2021-03-24T07:14:09.1622330Z     task = task_creator()
2021-03-24T07:14:09.1623410Z   File "/script-languages-release/exaslct_src/exaslct/cli/commands/build.py", line 73, in <lambda>
2021-03-24T07:14:09.1624380Z     task_creator = lambda: DockerBuild(flavor_paths=list(flavor_path), goals=list(goal), shortcut_build=shortcut_build)
2021-03-24T07:14:09.1626010Z   File "/home/vagrant/.local/share/virtualenvs/script-languages-release-igq1tTpT/lib/python3.6/site-packages/luigi/task_register.py", line 98, in __call__
2021-03-24T07:14:09.1626980Z     h[k] = instantiate()
2021-03-24T07:14:09.1628360Z   File "/home/vagrant/.local/share/virtualenvs/script-languages-release-igq1tTpT/lib/python3.6/site-packages/luigi/task_register.py", line 79, in instantiate
2021-03-24T07:14:09.1629440Z     return super(Register, cls).__call__(*args, **kwargs)
2021-03-24T07:14:09.1631060Z   File "/home/vagrant/.local/share/virtualenvs/script-languages-release-igq1tTpT/lib/python3.6/site-packages/exasol_integration_test_docker_environment/lib/base/flavor_task.py", line 14, in __init__
2021-03-24T07:14:09.1632180Z     super().__init__(*args, **kwargs)
2021-03-24T07:14:09.1634110Z   File "/home/vagrant/.local/share/virtualenvs/script-languages-release-igq1tTpT/lib/python3.6/site-packages/exasol_integration_test_docker_environment/lib/base/stoppable_base_task.py", line 15, in __init__
2021-03-24T07:14:09.1635310Z     super().__init__(*args, **kwargs)
2021-03-24T07:14:09.1636990Z   File "/home/vagrant/.local/share/virtualenvs/script-languages-release-igq1tTpT/lib/python3.6/site-packages/exasol_integration_test_docker_environment/lib/base/timeable_base_task.py", line 10, in __init__
2021-03-24T07:14:09.1638140Z     super().__init__(*args, **kwargs)
2021-03-24T07:14:09.1639680Z   File "/home/vagrant/.local/share/virtualenvs/script-languages-release-igq1tTpT/lib/python3.6/site-packages/exasol_integration_test_docker_environment/lib/base/base_task.py", line 85, in __init__
2021-03-24T07:14:09.1640810Z     self.register_required()
2021-03-24T07:14:09.1641960Z   File "/script-languages-release/exaslct_src/exaslct/lib/tasks/build/docker_build.py", line 15, in register_required
2021-03-24T07:14:09.1642930Z     tasks = self.create_tasks_for_flavors_with_common_params(DockerFlavorBuild)
2021-03-24T07:14:09.1644890Z   File "/home/vagrant/.local/share/virtualenvs/script-languages-release-igq1tTpT/lib/python3.6/site-packages/exasol_integration_test_docker_environment/lib/base/flavor_task.py", line 21, in create_tasks_for_flavors_with_common_params
2021-03-24T07:14:09.1646450Z     for flavor_path in self.flavor_paths}
2021-03-24T07:14:09.1648100Z   File "/home/vagrant/.local/share/virtualenvs/script-languages-release-igq1tTpT/lib/python3.6/site-packages/exasol_integration_test_docker_environment/lib/base/flavor_task.py", line 21, in <dictcomp>
2021-03-24T07:14:09.1649280Z     for flavor_path in self.flavor_paths}
2021-03-24T07:14:09.1650970Z   File "/home/vagrant/.local/share/virtualenvs/script-languages-release-igq1tTpT/lib/python3.6/site-packages/exasol_integration_test_docker_environment/lib/base/flavor_task.py", line 25, in _create_task_for_with_common_params
2021-03-24T07:14:09.1652270Z     task = self.create_child_task_with_common_params(cls, **params)
2021-03-24T07:14:09.1654020Z   File "/home/vagrant/.local/share/virtualenvs/script-languages-release-igq1tTpT/lib/python3.6/site-packages/exasol_integration_test_docker_environment/lib/base/base_task.py", line 302, in create_child_task_with_common_params
2021-03-24T07:14:09.1655220Z     return task_class(**params)
2021-03-24T07:14:09.1656600Z   File "/home/vagrant/.local/share/virtualenvs/script-languages-release-igq1tTpT/lib/python3.6/site-packages/luigi/task_register.py", line 98, in __call__
2021-03-24T07:14:09.1657540Z     h[k] = instantiate()
2021-03-24T07:14:09.1659250Z   File "/home/vagrant/.local/share/virtualenvs/script-languages-release-igq1tTpT/lib/python3.6/site-packages/luigi/task_register.py", line 79, in instantiate
2021-03-24T07:14:09.1660340Z     return super(Register, cls).__call__(*args, **kwargs)
2021-03-24T07:14:09.1662000Z   File "/home/vagrant/.local/share/virtualenvs/script-languages-release-igq1tTpT/lib/python3.6/site-packages/exasol_integration_test_docker_environment/lib/base/flavor_task.py", line 33, in __init__
2021-03-24T07:14:09.1663190Z     super().__init__(*args, **kwargs)
2021-03-24T07:14:09.1664780Z   File "/home/vagrant/.local/share/virtualenvs/script-languages-release-igq1tTpT/lib/python3.6/site-packages/exasol_integration_test_docker_environment/lib/base/docker_base_task.py", line 9, in __init__
2021-03-24T07:14:09.1665910Z     super().__init__(*args, **kwargs)
2021-03-24T07:14:09.1667560Z   File "/home/vagrant/.local/share/virtualenvs/script-languages-release-igq1tTpT/lib/python3.6/site-packages/exasol_integration_test_docker_environment/lib/docker/images/create/docker_build_base.py", line 18, in __init__
2021-03-24T07:14:09.1668730Z     super().__init__(*args, **kwargs)
2021-03-24T07:14:09.1670320Z   File "/home/vagrant/.local/share/virtualenvs/script-languages-release-igq1tTpT/lib/python3.6/site-packages/exasol_integration_test_docker_environment/lib/base/stoppable_base_task.py", line 15, in __init__
2021-03-24T07:14:09.1671450Z     super().__init__(*args, **kwargs)
2021-03-24T07:14:09.1673150Z   File "/home/vagrant/.local/share/virtualenvs/script-languages-release-igq1tTpT/lib/python3.6/site-packages/exasol_integration_test_docker_environment/lib/base/timeable_base_task.py", line 10, in __init__
2021-03-24T07:14:09.1674300Z     super().__init__(*args, **kwargs)
2021-03-24T07:14:09.1675990Z   File "/home/vagrant/.local/share/virtualenvs/script-languages-release-igq1tTpT/lib/python3.6/site-packages/exasol_integration_test_docker_environment/lib/base/base_task.py", line 85, in __init__
2021-03-24T07:14:09.1677110Z     self.register_required()
2021-03-24T07:14:09.1678830Z   File "/home/vagrant/.local/share/virtualenvs/script-languages-release-igq1tTpT/lib/python3.6/site-packages/exasol_integration_test_docker_environment/lib/docker/images/create/docker_build_base.py", line 33, in register_required
2021-03-24T07:14:09.1680300Z     self.analyze_tasks_futures = self.register_dependencies(self.create_analyze_tasks())
2021-03-24T07:14:09.1682250Z   File "/home/vagrant/.local/share/virtualenvs/script-languages-release-igq1tTpT/lib/python3.6/site-packages/exasol_integration_test_docker_environment/lib/docker/images/create/docker_build_base.py", line 47, in create_analyze_tasks
2021-03-24T07:14:09.1683630Z     difference = goals.difference(self.available_goals)
2021-03-24T07:14:09.1684640Z AttributeError: 'tuple' object has no attribute 'difference'
2021-03-24T07:14:09.2440090Z ##[error]Process completed with exit code 1.
```

https://github.com/exasol/integration-test-docker-environment/blob/19b409b8042d837f2e7e79c3e876ac9e35524a16/exasol_integration_test_docker_environment/lib/docker/images/create/docker_build_base.py#L47


## Fix

Use set(goals) instead goals which is a set for computing the difference 


Discussion / hints:

Fix patch:
diff --git a/.github/workflows/check_setup_py.yaml b/.github/workflows/check_setup_py.yaml
index 0d1e0c3d9..e98659b12 100644
--- a/.github/workflows/check_setup_py.yaml
+++ b/.github/workflows/check_setup_py.yaml
@@ -16,12 +16,14 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
     - uses: abatilo/actions-poetry@v2.0.0
+      with:
+        poetry-version: 1.1.6
     - name: Run packaging update
       run: bash githooks/update_packaging.sh
     - name: Show changes on working copy
       run: git status --porcelain=v1 -uno
     - name: Show diff on working copy
-      run: git diff
+      run: git diff --cached
     - name: Check if something changed after running the update script
       run: |
         [ -z "$(git status --porcelain=v1 -uno 2>/dev/null)" ] 
diff --git a/pyproject.toml b/pyproject.toml
index 97d8b0bf5..f9e7d27a9 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -16,6 +16,7 @@ homepage = "https://github.com/exasol/integration-test-docker-environment"
 
 keywords = ['exasol', 'docker', 'testing']
 include = ["docker_db_config","ext"]
+exclude = ["exasol_integration_test_docker_environment/test"]
 
 [tool.poetry.dependencies]
 python = ">=3.6"
diff --git a/setup.py b/setup.py
index 1363123ec..00d5d9467 100644
--- a/setup.py
+++ b/setup.py
@@ -12,6 +12,7 @@
  'exasol_integration_test_docker_environment.lib.data',
  'exasol_integration_test_docker_environment.lib.docker',
  'exasol_integration_test_docker_environment.lib.docker.images',
+ 'exasol_integration_test_docker_environment.lib.docker.images.clean',
  'exasol_integration_test_docker_environment.lib.docker.images.create',
  'exasol_integration_test_docker_environment.lib.docker.images.create.utils',
  'exasol_integration_test_docker_environment.lib.docker.images.push',
@@ -20,8 +21,7 @@
  'exasol_integration_test_docker_environment.lib.test_environment',
  'exasol_integration_test_docker_environment.lib.test_environment.database_setup',
  'exasol_integration_test_docker_environment.lib.test_environment.database_waiters',
- 'exasol_integration_test_docker_environment.lib.test_environment.parameter',
- 'exasol_integration_test_docker_environment.test']
+ 'exasol_integration_test_docker_environment.lib.test_environment.parameter']
 
 package_data = \
 {'': ['*'],


Test patch:
diff --git a/exasol_integration_test_docker_environment/lib/docker/images/clean/clean_images.py b/exasol_integration_test_docker_environment/lib/docker/images/clean/clean_images.py
new file mode 100644
index 000000000..8452f218c
--- /dev/null
+++ b/exasol_integration_test_docker_environment/lib/docker/images/clean/clean_images.py
@@ -0,0 +1,60 @@
+import luigi
+from exasol_integration_test_docker_environment.lib.base.docker_base_task import DockerBaseTask
+from exasol_integration_test_docker_environment.lib.config.docker_config import target_docker_repository_config
+from exasol_integration_test_docker_environment.lib.docker.images.utils import find_images_by_tag
+
+
+class CleanImageTask(DockerBaseTask):
+    jobid = luigi.Parameter("")
+    image_id = luigi.Parameter()
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def run_task(self):
+        self.logger.info("Try to remove dependent images of %s" % self.image_id)
+        yield from self.run_dependencies(self.get_clean_image_tasks_for_dependent_images())
+        for i in range(3):
+            try:
+                self.logger.info("Try to remove image %s" % self.image_id)
+                self._client.images.remove(image=self.image_id, force=True)
+                self.logger.info("Removed image %s" % self.image_id)
+                break
+            except Exception as e:
+                self.logger.info("Could not removed image %s got exception %s" % (self.image_id, e))
+
+    def get_clean_image_tasks_for_dependent_images(self):
+        image_ids = [str(possible_child).replace("sha256:", "") for possible_child
+                     in self._low_level_client.images(all=True, quiet=True)
+                     if self.is_child_image(possible_child)]
+        return [self.create_child_task(CleanImageTask, image_id=image_id)
+                for image_id in image_ids]
+
+    def is_child_image(self, possible_child):
+        try:
+            inspect = self._low_level_client.inspect_image(image=str(possible_child).replace("sha256:", ""))
+            return str(inspect["Parent"]).replace("sha256:", "") == self.image_id
+        except:
+            return False
+
+
+class CleanImagesStartingWith(DockerBaseTask):
+
+    jobid = luigi.Parameter("")
+    starts_with_pattern = luigi.Parameter()
+
+    def register_required(self):
+        image_ids = [str(image.id).replace("sha256:", "")
+                     for image in self.find_images_to_clean()]
+        self.register_dependencies([self.create_child_task(CleanImageTask, image_id=image_id)
+                                    for image_id in image_ids])
+
+    def find_images_to_clean(self):
+        self.logger.info("Going to remove all images starting with %s" % self.starts_with_pattern)
+        filter_images = find_images_by_tag(self._client, lambda tag: tag.startswith(self.starts_with_pattern))
+        for i in filter_images:
+            self.logger.info("Going to remove following image: %s" % i.tags)
+        return filter_images
+
+    def run_task(self):
+        pass
diff --git a/exasol_integration_test_docker_environment/lib/docker/images/create/docker_build_base.py b/exasol_integration_test_docker_environment/lib/docker/images/create/docker_build_base.py
index 4d31a2c95..ca790270e 100644
--- a/exasol_integration_test_docker_environment/lib/docker/images/create/docker_build_base.py
+++ b/exasol_integration_test_docker_environment/lib/docker/images/create/docker_build_base.py
@@ -33,7 +33,7 @@ def register_required(self):
         self.analyze_tasks_futures = self.register_dependencies(self.create_analyze_tasks())
 
     def create_analyze_tasks(self) -> Dict[str, DockerAnalyzeImageTask]:
-        goals = self.get_goals()
+        goals = set(self.get_goals())
         self.goal_class_map = self.get_goal_class_map()
         self.available_goals = set(self.goal_class_map.keys())
         self._check_if_build_steps_to_rebuild_are_valid_goals()
diff --git a/exasol_integration_test_docker_environment/lib/docker/images/utils.py b/exasol_integration_test_docker_environment/lib/docker/images/utils.py
new file mode 100644
index 000000000..e414b72a9
--- /dev/null
+++ b/exasol_integration_test_docker_environment/lib/docker/images/utils.py
@@ -0,0 +1,9 @@
+from typing import Callable, List
+
+
+def find_images_by_tag(client, condition: Callable[[str], bool]) -> List:
+    images = client.images.list()
+    filter_images = [image for image in images
+                     if image.tags is not None and len(image.tags) > 0 and
+                     any([condition(tag) for tag in image.tags])]
+    return filter_images
diff --git a/exasol_integration_test_docker_environment/test/resources/test_docker_build_base/test_analyze_image/Dockerfile b/exasol_integration_test_docker_environment/test/resources/test_docker_build_base/test_analyze_image/Dockerfile
new file mode 100644
index 000000000..617801170
--- /dev/null
+++ b/exasol_integration_test_docker_environment/test/resources/test_docker_build_base/test_analyze_image/Dockerfile
@@ -0,0 +1,1 @@
+FROM ubuntu:18.04
diff --git a/exasol_integration_test_docker_environment/test/test_docker_build_base.py b/exasol_integration_test_docker_environment/test/test_docker_build_base.py
new file mode 100644
index 000000000..445aa24f7
--- /dev/null
+++ b/exasol_integration_test_docker_environment/test/test_docker_build_base.py
@@ -0,0 +1,135 @@
+import shutil
+import time
+import unittest
+from pathlib import Path
+from datetime import datetime
+
+import docker
+import luigi
+from luigi import Parameter, Config
+
+from exasol_integration_test_docker_environment.lib.base.dependency_logger_base_task import DependencyLoggerBaseTask
+from exasol_integration_test_docker_environment.lib.base.json_pickle_parameter import JsonPickleParameter
+
+from typing import Set, Dict
+
+from exasol_integration_test_docker_environment.lib.config.docker_config import target_docker_repository_config, \
+    source_docker_repository_config
+from exasol_integration_test_docker_environment.lib.docker.images.create.docker_build_base import DockerBuildBase
+from exasol_integration_test_docker_environment.lib.docker.images.create.docker_image_analyze_task import \
+    DockerAnalyzeImageTask
+from exasol_integration_test_docker_environment.lib.docker.images.push.docker_push_parameter import DockerPushParameter
+from exasol_integration_test_docker_environment.lib.docker.images.push.push_task_creator_for_build_tasks import \
+    PushTaskCreatorFromBuildTasks
+from exasol_integration_test_docker_environment.lib.docker.images.clean.clean_images import CleanImagesStartingWith
+from exasol_integration_test_docker_environment.lib.docker.images.utils import find_images_by_tag
+
+class TestDockerBuildBaseTestAnalyzeImage(DockerAnalyzeImageTask):
+
+    task_name = Parameter()
+
+    def get_target_repository_name(self) -> str:
+        return f"""exasol-test-docker-build-base"""
+
+    def get_source_repository_name(self) -> str:
+        return f"""exasol-test-docker-build-base"""
+
+    def get_source_image_tag(self):
+        return self.task_name
+
+    def get_target_image_tag(self):
+        return self.task_name
+
+    def get_mapping_of_build_files_and_directories(self):
+        return {}
+
+    def get_dockerfile(self):
+        script_dir = Path(__file__).resolve().parent
+        dockerfile_path = Path(script_dir, "resources/test_docker_build_base/test_analyze_image/Dockerfile")
+        return dockerfile_path
+
+    def is_rebuild_requested(self) -> bool:
+        return False
+
+
+class TestDockerBuildBase(DockerBuildBase):
+    goals = luigi.ListParameter([])
+
+    def get_goal_class_map(self) -> Dict[str, DockerAnalyzeImageTask]:
+        goal_class_map = {
+                "test-analyze-image-1": TestDockerBuildBaseTestAnalyzeImage(task_name="test-analyze-image-1"),
+                "test-analyze-image-2": TestDockerBuildBaseTestAnalyzeImage(task_name="test-analyze-image-2")
+                }
+        return goal_class_map
+
+    def get_default_goals(self) -> Set[str]:
+        goals = {"test-analyze-image-1"}
+        return goals
+
+    def get_goals(self):
+        return self.goals
+
+    def run_task(self):
+        build_tasks = self.create_build_tasks(False)
+        image_infos_futures = yield from self.run_dependencies(build_tasks)
+        image_infos = self.get_values_from_futures(image_infos_futures)
+        self.return_object(image_infos)
+
+class DockerBuildBaseTest(unittest.TestCase):
+
+    def set_job_id(self, task_cls):
+        strftime = datetime.now().strftime('%Y_%m_%d_%H_%M_%S')
+        job_id = f"{strftime}_{task_cls.__name__}"
+        config = luigi.configuration.get_config()
+        config.set('job_config', 'job_id', job_id)
+        return job_id
+
+    def clean(self):
+        jobid = self.set_job_id(CleanImagesStartingWith)
+        task = CleanImagesStartingWith(starts_with_pattern="exasol-test-docker-build-base", jobid=jobid)
+        luigi.build([task], workers=1, local_scheduler=True, log_level="INFO")
+        if task._get_tmp_path_for_job().exists():
+            shutil.rmtree(str(task._get_tmp_path_for_job()))
+
+    def setUp(self):
+        self.client = docker.from_env()
+        self.clean()
+
+    def tearDown(self):
+        self.clean()
+        self.client.close()
+
+    def assert_image_exists(self, prefix):
+        image_list = find_images_by_tag(self.client, lambda x: x.startswith(prefix))
+        self.assertEquals(len(image_list),1,f"Image with prefix {prefix} not found")
+
+    def test_default_parameter(self):
+        self.set_job_id(TestDockerBuildBase)
+        task = TestDockerBuildBase()
+        try:
+            luigi.build([task], workers=1, local_scheduler=True, log_level="INFO")
+            self.assert_image_exists("exasol-test-docker-build-base:test-analyze-image-1")
+        finally:
+            if task._get_tmp_path_for_job().exists():
+                shutil.rmtree(str(task._get_tmp_path_for_job()))
+
+
+    def test_valid_non_default_goal(self):
+        self.set_job_id(TestDockerBuildBase)
+        task = TestDockerBuildBase(goals=["test-analyze-image-2"])
+        try:
+            luigi.build([task], workers=1, local_scheduler=True, log_level="INFO")
+            self.assert_image_exists("exasol-test-docker-build-base:test-analyze-image-2")
+        finally:
+            if task._get_tmp_path_for_job().exists():
+                shutil.rmtree(str(task._get_tmp_path_for_job()))
+
+    def test_non_valid_non_default_goal(self):
+        self.set_job_id(TestDockerBuildBase)
+        with self.assertRaises(Exception) as contex:
+            task = TestDockerBuildBase(goals=["test-analyze-image-3"])
+        self.assertIn("Unknown goal(s)",str(contex.exception))
+
+
+if __name__ == '__main__':
+    unittest.main()