Repository: easybuilders/easybuild-framework

Issue:
CUDA stubs directory must not be added to rpath/runpath
This is a follow up on https://github.com/easybuilders/easybuild-easyblocks/pull/1464 

 adding stubs to LIBRARY_PATH is causing us problems, because it ends up in our binaries RUNPATH.

If this happens and you don't use LD_LIBRARY_PATH (because you are using RUNPATH), you will have problems at runtime.

This is probably something you need to check when you are using runpath/rpath with easybuild (for Compute Canada, @bartoldeman had to adjust our linker wrapper to exclude these paths)

```
A stub object is a shared object, built entirely from mapfiles, that supplies the same linking interface as the real object, while containing no code or data. Stub objects cannot be used at runtime.
``` 




Discussion:
@mboisson How are these paths excluded in your linker wrapper exactly? Just based on directory name?
@bartoldeman would be best to answer that question. 
yes, as follows:
```
        # also exclude stub libraries like for CUDA
        if [ "${1%%/stubs}" != "$1" ]; then
            return 0
        fi
```
it's not beautiful, but it's unlikely that  a directory named "stubs" contains non-stub libraries I would think.
Are there any other cases known besides `CUDA`? If not, we can consider making the check a bit more strict to also check on `CUDA` in the path name...

Resolving patch:
diff --git a/easybuild/scripts/rpath_args.py b/easybuild/scripts/rpath_args.py
--- a/easybuild/scripts/rpath_args.py
+++ b/easybuild/scripts/rpath_args.py
@@ -98,12 +98,13 @@
             cmd_args.append('-L%s' % lib_path)
 
     # replace --enable-new-dtags with --disable-new-dtags if it's used;
-    # --enable-new-dtags would result in copying rpath to runpath, meaning that $LD_LIBRARY_PATH is taken into account again
-    # --enable-new-dtags is not removed but replaced to prevent issues when the linker flag is forwarded from the compiler 
-    # to the linker with an extra prefixed flag (either -Xlinker or -Wl,). In that case, the compiler would erroneously pass
-    # the next random argument to the linker.
+    # --enable-new-dtags would result in copying rpath to runpath,
+    # meaning that $LD_LIBRARY_PATH is taken into account again;
+    # --enable-new-dtags is not removed but replaced to prevent issues when linker flag is forwarded from the compiler
+    # to the linker with an extra prefixed flag (either -Xlinker or -Wl,).
+    # In that case, the compiler would erroneously pass the next random argument to the linker.
     elif arg == '--enable-new-dtags':
-        cmd_args.append('--disable-new-dtags');
+        cmd_args.append('--disable-new-dtags')
     else:
         cmd_args.append(arg)
 
@@ -121,7 +122,7 @@
     ] + cmd_args
 
 # wrap all arguments into single quotes to avoid further bash expansion
-cmd_args = ["'%s'" % arg.replace("'", "''") for arg in cmd_args]
+cmd_args = ["'%s'" % a.replace("'", "''") for a in cmd_args]
 
 # output: statement to define $CMD_ARGS and $RPATH_ARGS
 print("CMD_ARGS=(%s)" % ' '.join(cmd_args))
diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py
--- a/easybuild/tools/toolchain/toolchain.py
+++ b/easybuild/tools/toolchain/toolchain.py
@@ -819,6 +819,13 @@ def prepare_rpath_wrappers(self, rpath_filter_dirs=None, rpath_include_dirs=None
         else:
             raise EasyBuildError("RPATH linking is currently only supported on Linux")
 
+        if rpath_filter_dirs is None:
+            rpath_filter_dirs = []
+
+        # always include filter for 'stubs' library directory,
+        # cfr. https://github.com/easybuilders/easybuild-framework/issues/2683
+        rpath_filter_dirs.append('.*/lib(64)?/stubs/?')
+
         # directory where all wrappers will be placed
         wrappers_dir = os.path.join(tempfile.mkdtemp(), RPATH_WRAPPERS_SUBDIR)
 
@@ -833,7 +840,7 @@ def prepare_rpath_wrappers(self, rpath_filter_dirs=None, rpath_include_dirs=None
         if rpath_filter is None:
             rpath_filter = ['/lib.*', '/usr.*']
             self.log.debug("No general RPATH filter specified, falling back to default: %s", rpath_filter)
-        rpath_filter = ','.join(rpath_filter + ['%s.*' % d for d in rpath_filter_dirs or []])
+        rpath_filter = ','.join(rpath_filter + ['%s.*' % d for d in rpath_filter_dirs])
         self.log.debug("Combined RPATH filter: '%s'", rpath_filter)
 
         rpath_include = ','.join(rpath_include_dirs or [])


Test patch:
diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py
--- a/test/framework/toolchain.py
+++ b/test/framework/toolchain.py
@@ -1403,7 +1403,8 @@ def test_toolchain_prepare_rpath(self):
 
         # check whether fake g++ was wrapped and that arguments are what they should be
         # no -rpath for /bar because of rpath filter
-        out, _ = run_cmd('g++ ${USER}.c -L/foo -L/bar \'$FOO\' -DX="\\"\\""')
+        out, ec = run_cmd('g++ ${USER}.c -L/foo -L/bar \'$FOO\' -DX="\\"\\""')
+        self.assertEqual(ec, 0)
         expected = ' '.join([
             '-Wl,--disable-new-dtags',
             '-Wl,-rpath=/foo',
@@ -1415,6 +1416,44 @@ def test_toolchain_prepare_rpath(self):
         ])
         self.assertEqual(out.strip(), expected % {'user': os.getenv('USER')})
 
+        # check whether 'stubs' library directory are correctly filtered out
+        args = [
+            '-L/prefix/software/CUDA/1.2.3/lib/stubs/',  # should be filtered (no -rpath)
+            '-L/tmp/foo/',
+            '-L/prefix/software/stubs/1.2.3/lib',  # should NOT be filtered
+            '-L/prefix/software/CUDA/1.2.3/lib/stubs',  # should be filtered (no -rpath)
+            '-L/prefix/software/CUDA/1.2.3/lib64/stubs/',  # should be filtered (no -rpath)
+            '-L/prefix/software/foobar/4.5/notreallystubs',  # should NOT be filtered
+            '-L/prefix/software/CUDA/1.2.3/lib64/stubs',  # should be filtered (no -rpath)
+            '-L/prefix/software/zlib/1.2.11/lib',
+            '-L/prefix/software/bleh/0/lib/stubs',  # should be filtered (no -rpath)
+            '-L/prefix/software/foobar/4.5/stubsbutnotreally',  # should NOT be filtered
+        ]
+        cmd = "g++ ${USER}.c %s" % ' '.join(args)
+        out, ec = run_cmd(cmd, simple=False)
+        self.assertEqual(ec, 0)
+
+        expected = ' '.join([
+            '-Wl,--disable-new-dtags',
+            '-Wl,-rpath=/tmp/foo/',
+            '-Wl,-rpath=/prefix/software/stubs/1.2.3/lib',
+            '-Wl,-rpath=/prefix/software/foobar/4.5/notreallystubs',
+            '-Wl,-rpath=/prefix/software/zlib/1.2.11/lib',
+            '-Wl,-rpath=/prefix/software/foobar/4.5/stubsbutnotreally',
+            '%(user)s.c',
+            '-L/prefix/software/CUDA/1.2.3/lib/stubs/',
+            '-L/tmp/foo/',
+            '-L/prefix/software/stubs/1.2.3/lib',
+            '-L/prefix/software/CUDA/1.2.3/lib/stubs',
+            '-L/prefix/software/CUDA/1.2.3/lib64/stubs/',
+            '-L/prefix/software/foobar/4.5/notreallystubs',
+            '-L/prefix/software/CUDA/1.2.3/lib64/stubs',
+            '-L/prefix/software/zlib/1.2.11/lib',
+            '-L/prefix/software/bleh/0/lib/stubs',
+            '-L/prefix/software/foobar/4.5/stubsbutnotreally',
+        ])
+        self.assertEqual(out.strip(), expected % {'user': os.getenv('USER')})
+
         # calling prepare() again should *not* result in wrapping the existing RPATH wrappers
         # this can happen when building extensions
         tc.prepare()