Repository: easybuilders/easybuild-framework

Issue:
eb does not automatically module Core modules during a robot build
```
buildpath            (D) = /afs/cad/u/g/w/xxx/.local/easybuild/build
installpath          (D) = /afs/cad/u/g/w/xxx/.local/easybuild
module-naming-scheme (E) = HierarchicalMNS
repositorypath       (D) = /afs/cad/u/g/w/xxx/.local/easybuild/ebfiles_repo
 robot-paths          (D) = afs/cad/u/g/w/xxx/eb/software/Core/EasyBuild/3.1.2/lib/python2.7/site-packages/easybuild_easyconfigs-3.1.2-py2.7.egg/easybuild/easyconfigs
 sourcepath           (D) = /afs/cad/u/g/w/xxx/.local/easybuild/sources
```

`eb GCC-6.3.0-2.27.eb --robot --installpath=XXXX --installpath-modules =YYYY --robot `
Bison installation failed due to a module load issue.
From the log --

```
 build_log.py:147 ERROR EasyBuild crashed with an error (at ?:124 in __init__): Changing environment as dictated by module failed: name 'false' is
os.environ["MODULEPATH"] = "/afs/cad/u/g/w/XXX/.local/easybuild/modules/all:/etc/modulefiles:/usr/share/modulefiles:/usr/share/modulefiles/Linux:/usr/share/modulefiles/C
false
, stderr: Lmod has detected the following error: The following module(s) are unknown:
"M4/1.4.17"
```

module use YYYY/all/Core and rerun - Builds successfully


Discussion:
@geimer Thoughts on this? Should EasyBuild do the `module use .../Core` itself, maybe along with a big fat warning that `$MODULEPATH` isn't set up correctly to consume the modules EasyBuild procudes in the HMNS?
@boegel I never tried `eb` with `$MODULEPATH` not being set up correctly.  What does EasyBuild do for the "classical" naming scheme?  Will `modules/all` be added to `$MODULEPATH` automatically?  If so, I think that `Core` should be added when using HMNS.
@geimer To get back to this: yes, EasyBuild automatically makes sure that the location where it installs modules too is prepended to `$MODULEPATH.

I agree it makes sense to do the same for an HMNS...
@boegel So if I'm using HMNS I have to add those paths manually to the MODULEPATH? 
This confuses me as det_init_modulepaths returns exactly those missing paths, is there no way to automate this?

Resolving patch:
diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -87,8 +87,8 @@
 from easybuild.tools.module_generator import ModuleGeneratorLua, ModuleGeneratorTcl, module_generator, dependencies_for
 from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version
 from easybuild.tools.modules import ROOT_ENV_VAR_NAME_PREFIX, VERSION_ENV_VAR_NAME_PREFIX, DEVEL_ENV_VAR_NAME_PREFIX
-from easybuild.tools.modules import invalidate_module_caches_for, get_software_root, get_software_root_env_var_name
-from easybuild.tools.modules import get_software_version_env_var_name
+from easybuild.tools.modules import curr_module_paths, invalidate_module_caches_for, get_software_root
+from easybuild.tools.modules import get_software_root_env_var_name, get_software_version_env_var_name
 from easybuild.tools.package.utilities import package
 from easybuild.tools.repository.repository import init_repository
 from easybuild.tools.toolchain import DUMMY_TOOLCHAIN_NAME
@@ -1924,6 +1924,16 @@ def prepare_step(self, start_dir=True):
             # reset toolchain for iterative runs before preparing it again
             self.toolchain.reset()
 
+        # if active module naming scheme involves any top-level directories in the hierarchy (e.g. Core/ in HMNS)
+        # make sure they are included in $MODULEPATH such that loading of dependencies (with short module names) works
+        # https://github.com/easybuilders/easybuild-framework/issues/2186
+        init_modpaths = ActiveMNS().det_init_modulepaths(self.cfg)
+        curr_modpaths = curr_module_paths()
+        for init_modpath in init_modpaths:
+            full_mod_path = os.path.join(self.installdir_mod, init_modpath)
+            if full_mod_path not in curr_modpaths:
+                self.modules_tool.prepend_module_path(full_mod_path)
+
         # prepare toolchain: load toolchain module and dependencies, set up build environment
         self.toolchain.prepare(self.cfg['onlytcmod'], deps=self.cfg.dependencies(), silent=self.silent,
                                rpath_filter_dirs=self.rpath_filter_dirs, rpath_include_dirs=self.rpath_include_dirs)


Test patch:
diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py
--- a/test/framework/easyblock.py
+++ b/test/framework/easyblock.py
@@ -1450,6 +1450,43 @@ def test_prepare_step(self):
         self.assertEqual(len(self.modtool.list()), 1)
         self.assertEqual(self.modtool.list()[0]['mod_name'], 'GCC/6.4.0-2.28')
 
+    def test_prepare_step_hmns(self):
+        """
+        Check whether loading of already existing dependencies during prepare step works when HierarchicalMNS is used.
+        """
+        test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
+
+        os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'HierarchicalMNS'
+        init_config(build_options={'robot_path': [test_ecs]})
+
+        # set up hierarchical modules, but reset $MODULEPATH to empty
+        # the expectation is that EasyBuild set's up the $MODULEPATH such that pre-installed dependencies can be loaded
+        # see also https://github.com/easybuilders/easybuild-framework/issues/2186
+        self.setup_hierarchical_modules()
+
+        self.assertTrue('GCC/6.4.0-2.28' in self.modtool.available())
+
+        self.reset_modulepath([])
+        self.assertEqual(os.environ.get('MODULEPATH'), None)
+
+        toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb')
+
+        test_ec = os.path.join(self.test_prefix, 'test.eb')
+        regex = re.compile('^toolchain = .*', re.M)
+        test_ectxt = regex.sub("toolchain = {'name': 'dummy', 'version': ''}", read_file(toy_ec))
+        test_ectxt += "\ndependencies = [('GCC', '6.4.0', '-2.28')]"
+        write_file(test_ec, test_ectxt)
+
+        test_ec = process_easyconfig(test_ec)[0]
+        eb = EasyBlock(test_ec['ec'])
+
+        mkdir(os.path.join(self.test_buildpath, 'toy', '0.0', 'dummy-'), parents=True)
+        eb.prepare_step()
+
+        loaded_modules = self.modtool.list()
+        self.assertEqual(len(loaded_modules), 1)
+        self.assertEqual(loaded_modules[0]['mod_name'], 'GCC/6.4.0-2.28')
+
     def test_checksum_step(self):
         """Test checksum step"""
         testdir = os.path.abspath(os.path.dirname(__file__))