Repository: exasol/integration-test-docker-environment (language: python)

Issue:
EXASLCT is caught in an infinte loop in case of symlink loops
## Situation

- Bazel creates during a build symlinks to the result of the build,  but also to the source directory. This symlink to the source directory is a symlink loop (circular symlink). 
- If you build the exaudfclient locally with bazel for testing and then try to build a flavor, EXASLCT is caught in an infinite loop which is caused by this symlink loop.
- EXASLCT is accumulating more and more memory and fails at some point with an out of memory error.

## Workaround

- Use `bazel clean` to remove the symlinks or use the build configs `--config no-symlinks` or `--config symlinks-to-tmp` which are defined in the .bazelrc of the exaudfclient

## Solution

- We could exclude the bazel symlinks from the builds with exasol/script-languages-release#143 
- Or, a proper solution would be the detection of the symlink loop. You might be able to do this by collecting the already seen inodes of symlinks to directories and if you encounter one twice, throw an exception.
- Alternative, count the depths of your recursion and break after a configurable depth

The fix needs to be done in the https://github.com/exasol/integration-test-docker-environment/blob/master/exasol_integration_test_docker_environment/lib/docker/images/create/utils/file_directory_list_hasher.py

Check number of path segments at https://github.com/exasol/integration-test-docker-environment/blob/948ed9bb5e356c78057f92a8512353d107a1d085/exasol_integration_test_docker_environment/lib/docker/images/create/utils/file_directory_list_hasher.py#L122, os.walk doesn't provide depth in another form


Discussion / hints:

Fix patch:
diff --git a/doc/changes/changes_0.4.0.md b/doc/changes/changes_0.4.0.md
index 72576eadc..8f402f422 100644
--- a/doc/changes/changes_0.4.0.md
+++ b/doc/changes/changes_0.4.0.md
@@ -20,6 +20,7 @@ If you need further versions, please open an issue.
 
 ## Bug Fixes:
    - #99: Fixed click parameters for external-exasol-db-port and external-exasol-bucketfs-port
+   - #97: Fixed computation of hash in exaslct if path contains symlink loops
 
 ## Features / Enhancements:
     


Test patch:
diff --git a/exasol_integration_test_docker_environment/lib/docker/images/create/utils/file_directory_list_hasher.py b/exasol_integration_test_docker_environment/lib/docker/images/create/utils/file_directory_list_hasher.py
index 8e099a40c..5029770ea 100644
--- a/exasol_integration_test_docker_environment/lib/docker/images/create/utils/file_directory_list_hasher.py
+++ b/exasol_integration_test_docker_environment/lib/docker/images/create/utils/file_directory_list_hasher.py
@@ -28,9 +28,10 @@ def __init__(self,
                  excluded_files=None,
                  excluded_extensions=None,
                  blocksize: str = "64kb",
-                 workers: int = 4
+                 workers: int = 4,
+                 max_characters_paths: int = 500000000
                  ):
-
+        self.MAX_CHARACTERS_PATHS = max_characters_paths
         self.use_relative_paths = use_relative_paths
         self.workers = workers
         self.excluded_files = excluded_files
@@ -118,14 +119,29 @@ def collect_files_and_directories(
         tmp_collected_files = []
         if self.hash_directory_names:
             tmp_collected_directories.append((directory, directory))
+        inodes = set()
+        numCharacters = 0
+
         for root, dirs, files in os.walk(directory, topdown=True, followlinks=self.followlinks):
+            stat = os.stat(root)
+            if stat.st_ino > 0 and stat.st_ino in inodes:
+                raise OSError(
+                    f"Directory: {directory} contains symlink loops (Symlinks pointing to a parent directory). Please fix!")
+            inodes.add(stat.st_ino)
+
             if self.hash_directory_names:
-                tmp_collected_directories.extend(
-                    [(directory, os.path.join(root, f)) for f in dirs
-                     if not self.is_excluded_directory(f)])
-            tmp_collected_files.extend(
-                [(directory, os.path.join(root, f)) for f in files
-                 if not self.is_excluded_file(f) and not self.has_excluded_extension(f)])
+                new_directories = [d for d in dirs if not self.is_excluded_directory(d)]
+                tmp_collected_directories.extend([(directory, os.path.join(root, d)) for d in new_directories])
+                numCharacters += sum([len(d[1]) for d in new_directories])
+
+            new_files = [f for f in files
+                         if not self.is_excluded_file(f) and not self.has_excluded_extension(f)]
+            tmp_collected_files.extend([(directory, os.path.join(root, f)) for f in new_files])
+            numCharacters += sum([len(f[1]) for f in new_files])
+
+            if numCharacters > self.MAX_CHARACTERS_PATHS:
+                raise OSError(f"Walking through too many directories. Aborting. Please verify: {directory}")
+
         collected_directories.extend(sorted(tmp_collected_directories, key=lambda x: x[1]))
         collected_files.extend(sorted(tmp_collected_files, key=lambda x: x[1]))
 
diff --git a/exasol_integration_test_docker_environment/test/test_symlink_loops.py b/exasol_integration_test_docker_environment/test/test_symlink_loops.py
new file mode 100644
index 000000000..c5579c39c
--- /dev/null
+++ b/exasol_integration_test_docker_environment/test/test_symlink_loops.py
@@ -0,0 +1,90 @@
+import os
+import shutil
+import unittest
+import tempfile
+
+from exasol_integration_test_docker_environment.lib.docker.images.create.utils.file_directory_list_hasher import \
+    FileDirectoryListHasher
+
+
+class TestSymlinkLoops(unittest.TestCase):
+    def setUp(self):
+        self.temp_dir = tempfile.mkdtemp() + "/" + self.__class__.__name__
+        self.temp_dir_dummy = tempfile.mkdtemp()
+
+    def generate_test_dir(self, with_symlink_loop):
+        level1 = 5
+        level2 = 5
+        level3 = 5
+        level4 = 5
+        level5 = 5
+        for i1 in range(level1):
+            for i2 in range(level2):
+                for i3 in range(level3):
+                    for i4 in range(level4):
+                        path = "/level0/level1_%s/level2_%s/level3_%s/level4_%s/" \
+                               % (i1, i2, i3, i4)
+                        os.makedirs(self.temp_dir + path)
+                        os.makedirs(self.temp_dir + path + "test")
+                        for i5 in range(level5):
+                            file = "%s/level5_file_%s" % (path, i5)
+                            with open(self.temp_dir + file, mode="wt") as f:
+                                f.write(file)
+
+        path = "/level0/level1_%s/level2_%s/level3_%s/level4_%s/" \
+               % (level1-1, level2-1, level3-1, level4-1)
+        dest_path = "/level0/level1_%s" \
+               % (level1-1)
+
+        if with_symlink_loop:
+            os.symlink(self.temp_dir + dest_path, self.temp_dir + f"{path}/evil_symlink")
+        return self.temp_dir + f"{path}/evil_symlink"
+
+    def tearDown(self):
+        shutil.rmtree(self.temp_dir)
+        shutil.rmtree(self.temp_dir_dummy)
+
+    def test_symlink_detection(self):
+        self.generate_test_dir(True)
+        hasher = FileDirectoryListHasher(hashfunc="sha256",
+                                         use_relative_paths=False,
+                                         hash_directory_names=True,
+                                         hash_file_names=True,
+                                         followlinks=True)
+        exception_thrown = False
+        try:
+            hash = hasher.hash([self.temp_dir])
+        except OSError as e:
+            if "contains symlink loops" in str(e):
+                exception_thrown = True
+        assert exception_thrown
+
+    def test_symlink_detection_size(self):
+        self.generate_test_dir(True)
+        hasher = FileDirectoryListHasher(hashfunc="sha256",
+                                         use_relative_paths=False,
+                                         hash_directory_names=True,
+                                         hash_file_names=True,
+                                         followlinks=True,
+                                         max_characters_paths=100)
+        exception_thrown = False
+        try:
+            hash = hasher.hash([self.temp_dir])
+        except OSError as e:
+            if "Walking through too many directories." in str(e):
+                exception_thrown = True
+        assert exception_thrown
+
+    def test_symlink_no_loop(self):
+        symlink_dest = self.generate_test_dir(False)
+        os.symlink(self.temp_dir_dummy, symlink_dest)
+        hasher = FileDirectoryListHasher(hashfunc="sha256",
+                                         use_relative_paths=False,
+                                         hash_directory_names=True,
+                                         hash_file_names=True,
+                                         followlinks=True)
+        hash = hasher.hash([self.temp_dir])
+
+
+if __name__ == '__main__':
+    unittest.main()