Repository: easybuilders/easybuild-framework

Issue:
get_software_libdir fails for symlinked lib64->lib
An installation of SuperLU fails due to it calling `openblas_lib = os.path.join(get_software_root('OpenBLAS'), get_software_libdir('OpenBLAS'), "libopenblas.a")` with an implicit `only_one=True` parameter to `get_software_libdir` but that finds `lib` and `lib64` and fails with an error.

The problem is that (our?) installation of `OpenBLAS/0.2.20-GCC-6.4.0-2.28` contains a symlink `lib64 -> lib`.

There are multiple solutions to this:
1. Reinstall OpenBLAS w/o the symlink. I guess it was manually added because I can't see it in the reinstalled one. But maybe this makes something else fail
2. SuperLU should be a) using `get_software_libdir('OpenBLAS', fs=["libopenblas.a"]) and b) then allow multiple dirs to be returned and simply pick the first
3. `get_software_libdir` should not return/check paths which are the same real path (resolving symlinks). I'd prefer this as this is the more universal solution that avoids problems in other EBs


Discussion:
@Flamefire Can you provide some more details on how the `SuperLU` installation is failing exactly?

`get_software_libdir` basically just does `os.path.exists`, it's not actively trying to resolve symlinks at all...

Resolving patch:
diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py
--- a/easybuild/tools/modules.py
+++ b/easybuild/tools/modules.py
@@ -1430,9 +1430,17 @@ def get_software_libdir(name, only_one=True, fs=None):
     res = []
     if root:
         for lib_subdir in lib_subdirs:
-            if os.path.exists(os.path.join(root, lib_subdir)):
-                if fs is None or any([os.path.exists(os.path.join(root, lib_subdir, f)) for f in fs]):
+            lib_dir_path = os.path.join(root, lib_subdir)
+            if os.path.exists(lib_dir_path):
+                # take into account that lib64 could be a symlink to lib (or vice versa)
+                # see https://github.com/easybuilders/easybuild-framework/issues/3139
+                if any(os.path.samefile(lib_dir_path, os.path.join(root, x)) for x in res):
+                    _log.debug("%s is the same as one of the other paths, so skipping it", lib_dir_path)
+
+                elif fs is None or any(os.path.exists(os.path.join(lib_dir_path, f)) for f in fs):
+                    _log.debug("Retaining library subdir '%s' (found at %s)", lib_subdir, lib_dir_path)
                     res.append(lib_subdir)
+
             elif build_option('extended_dry_run'):
                 res.append(lib_subdir)
                 break


Test patch:
diff --git a/test/framework/modules.py b/test/framework/modules.py
--- a/test/framework/modules.py
+++ b/test/framework/modules.py
@@ -43,7 +43,8 @@
 import easybuild.tools.modules as mod
 from easybuild.framework.easyblock import EasyBlock
 from easybuild.tools.build_log import EasyBuildError
-from easybuild.tools.filetools import adjust_permissions, copy_file, copy_dir, mkdir, read_file, remove_file, write_file
+from easybuild.tools.filetools import adjust_permissions, copy_file, copy_dir, mkdir
+from easybuild.tools.filetools import read_file, remove_dir, remove_file, symlink, write_file
 from easybuild.tools.modules import EnvironmentModules, EnvironmentModulesC, EnvironmentModulesTcl, Lmod, NoModulesTool
 from easybuild.tools.modules import curr_module_paths, get_software_libdir, get_software_root, get_software_version
 from easybuild.tools.modules import invalidate_module_caches_for, modules_tool, reset_module_caches
@@ -462,7 +463,7 @@ def test_get_software_root_version_libdir(self):
         for (name, env_var_name) in test_cases:
             # mock stuff that get_software_X functions rely on
             root = os.path.join(tmpdir, name)
-            os.makedirs(os.path.join(root, 'lib'))
+            mkdir(os.path.join(root, 'lib'), parents=True)
             os.environ['EBROOT%s' % env_var_name] = root
             version = '0.0-%s' % root
             os.environ['EBVERSION%s' % env_var_name] = version
@@ -476,7 +477,7 @@ def test_get_software_root_version_libdir(self):
 
         # check expected result of get_software_libdir with multiple lib subdirs
         root = os.path.join(tmpdir, name)
-        os.makedirs(os.path.join(root, 'lib64'))
+        mkdir(os.path.join(root, 'lib64'))
         os.environ['EBROOT%s' % env_var_name] = root
         self.assertErrorRegex(EasyBuildError, "Multiple library subdirectories found.*", get_software_libdir, name)
         self.assertEqual(get_software_libdir(name, only_one=False), ['lib', 'lib64'])
@@ -485,6 +486,19 @@ def test_get_software_root_version_libdir(self):
         open(os.path.join(root, 'lib64', 'foo'), 'w').write('foo')
         self.assertEqual(get_software_libdir(name, fs=['foo']), 'lib64')
 
+        # duplicate paths due to symlink get filtered
+        remove_dir(os.path.join(root, 'lib64'))
+        symlink(os.path.join(root, 'lib'), os.path.join(root, 'lib64'))
+        self.assertEqual(get_software_libdir(name), 'lib')
+
+        # same goes for lib symlinked to lib64
+        remove_file(os.path.join(root, 'lib64'))
+        remove_dir(os.path.join(root, 'lib'))
+        mkdir(os.path.join(root, 'lib64'))
+        symlink(os.path.join(root, 'lib64'), os.path.join(root, 'lib'))
+        # still returns 'lib' because that's the first subdir considered
+        self.assertEqual(get_software_libdir(name), 'lib')
+
         # clean up for previous tests
         os.environ.pop('EBROOT%s' % env_var_name)