Repository: easybuilders/easybuild-framework

Issue:
Build succesfully proceed regardless of error in get_software_version() with iimpi toolchain
I'm seeing this error at the very beginning of the build with anything in `iimpi` on current `develop`. However, the build just proceeds and finishes successfully.

>== 2020-09-11 10:55:21,934 build_log.py:169 ERROR EasyBuild crashed with an error (at easybuild/base/exceptions.py:124 in __init__): get_software_version software version for impi was not found in environment (at easybuild/tools/toolchain/toolchain.py:432 in _get_software_version)
== 2020-09-11 10:55:21,935 templates.py:301 WARNING Failed to create mpi_cmd_prefix template, error was:
'get_software_version software version for impi was not found in environment'

Example easyconfigs:
* HDF5-1.10.5-iimpi-2019b.eb
* Boost-1.70.0-iimpi-2019a.eb : [full log](https://gist.github.com/61f327aa4d13798b15850567abda8c5b)
* netCDF-4.7.1-iimpi-2019b.eb : [full log](https://gist.github.com/7e3f9d1dac87f579bc48f68e1646bd81)


Discussion:

Resolving patch:
diff --git a/easybuild/tools/toolchain/mpi.py b/easybuild/tools/toolchain/mpi.py
--- a/easybuild/tools/toolchain/mpi.py
+++ b/easybuild/tools/toolchain/mpi.py
@@ -258,10 +258,15 @@ def mpi_family(self):
     def mpi_cmd_prefix(self, nr_ranks=1):
         """Construct an MPI command prefix to precede an executable"""
 
-        # Verify that the command appears at the end of mpi_cmd_for
         test_cmd = 'xxx_command_xxx'
         mpi_cmd = self.mpi_cmd_for(test_cmd, nr_ranks)
-        if mpi_cmd.rstrip().endswith(test_cmd):
+
+        # take into account that result from mpi_cmd_for may be None,
+        # for example when it's called too early (before toolchain module is loaded)
+        if mpi_cmd is None:
+            result = None
+        # verify that the command appears at the end of mpi_cmd_for
+        elif mpi_cmd.rstrip().endswith(test_cmd):
             result = mpi_cmd.replace(test_cmd, '').rstrip()
         else:
             warning_msg = "mpi_cmd_for cannot be used by mpi_cmd_prefix, "
@@ -282,10 +287,19 @@ def mpi_cmd_for(self, cmd, nr_ranks):
 
         mpi_family = self.mpi_family()
 
+        mpi_version = None
+
         if mpi_family == toolchain.INTELMPI:
-            mpi_version = self.get_software_version(self.MPI_MODULE_NAME)[0]
-        else:
-            mpi_version = None
+            # for Intel MPI, try to determine impi version
+            # this fails when it's done too early (before modules for toolchain/dependencies are loaded),
+            # but it's safe to ignore this
+            try:
+                mpi_version = self.get_software_version(self.MPI_MODULE_NAME)[0]
+            except EasyBuildError as err:
+                self.log.debug("Ignoring error when trying to determine %s version: %s", self.MPI_MODULE_NAME, err)
+                # impi version is required to determine correct MPI command template,
+                # so we have to return early if we couldn't determine the impi version...
+                return None
 
         mpi_cmd_template, params = get_mpi_cmd_template(mpi_family, params, mpi_version=mpi_version)
         self.log.info("Using MPI command template '%s' (params: %s)", mpi_cmd_template, params)


Test patch:
diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py
--- a/test/framework/toolchain.py
+++ b/test/framework/toolchain.py
@@ -1197,6 +1197,16 @@ def test_mpi_cmd_prefix(self):
         """Test mpi_exec_nranks function."""
         self.modtool.prepend_module_path(self.test_prefix)
 
+        # first try calling mpi_cmd_prefix without having any modules loaded, should not cause trouble
+
+        # for toolchain with Intel MPI we get None as result in this cause (because impi version can
+        # not be determined)
+        tc = self.get_toolchain('intel', version='2018a')
+        self.assertEqual(tc.mpi_cmd_prefix(nr_ranks=2), None)
+
+        tc = self.get_toolchain('gompi', version='2018a')
+        self.assertEqual(tc.mpi_cmd_prefix(nr_ranks=2), "mpirun -n 2")
+
         tc = self.get_toolchain('gompi', version='2018a')
         tc.prepare()
         self.assertEqual(tc.mpi_cmd_prefix(nr_ranks=2), "mpirun -n 2")
@@ -1239,6 +1249,16 @@ def test_mpi_cmd_for(self):
         """Test mpi_cmd_for function."""
         self.modtool.prepend_module_path(self.test_prefix)
 
+        # if mpi_cmd_for is called too early when using toolchain that includes impi,
+        # we get None as result because Intel MPI version can not be determined
+        tc = self.get_toolchain('intel', version='2018a')
+        self.assertEqual(tc.mpi_cmd_for('test123', 2), None)
+
+        # no problem for OpenMPI-based toolchain, because OpenMPI version is not required
+        # to determine MPI command template
+        tc = self.get_toolchain('gompi', version='2018a')
+        self.assertEqual(tc.mpi_cmd_for('test123', 2), "mpirun -n 2 test123")
+
         tc = self.get_toolchain('gompi', version='2018a')
         tc.prepare()
         self.assertEqual(tc.mpi_cmd_for('test123', 2), "mpirun -n 2 test123")