Repository: easybuilders/easybuild-framework

Issue:
Occasional failures with Python 2, triggered by cleanup of imported modules in verify_imports
When using EasyBuild with Python 2, I'm experiencing occasional failures with a traceback like this:
```
[...]
  File ".../easybuild/framework/easyconfig/easyconfig.py", line 2537, in det_full_module_name
    require_result=require_result)
  File ".../easybuild/framework/easyconfig/easyconfig.py", line 2509, in _det_module_name_with
    mod_name = mns_method(ec)
  File ".../easybuild/tools/module_naming_scheme/hierarchical_mns.py", line 89, in det_full_module_name
    return os.path.join(self.det_module_subdir(ec), self.det_short_module_name(ec))
AttributeError: 'NoneType' object has no attribute 'path'
```
The exact circumstances still seem to be unclear, but according to @Flamefire it may be related to https://stackoverflow.com/questions/29107470/imported-modules-becomes-none-when-replacing-current-module-in-sys-modules-using


Discussion:
In short: Unloading modules is not supported in Python: https://bugs.python.org/issue9072
More general: `sys.modules` mustn't be modified manually. But we do that in https://github.com/easybuilders/easybuild-framework/blob/b7031917445a536e591f593d1661168bd8cc495c/easybuild/tools/include.py#L146-L147

This is fatal in Python < 3.4 which clears the modules dict on unload: https://stackoverflow.com/questions/29107470/imported-modules-becomes-none-when-replacing-current-module-in-sys-modules-using and https://stackoverflow.com/questions/29107470/imported-modules-becomes-none-when-replacing-current-module-in-sys-modules-using

In Python 3.4+ the unloading seems to happen differently (or maybe even work at all). But I don't see the reasoning here to unload them, it just leads to more work loading them later again

Solutions:

- Don't modify `sys.modules`, i.e. keep the checked modules loaded
- Skip the check (do nothing in `verify_imports`) for Python < 3.4
- Remove that whole function
@geimer I think you mentioned that this issue is new, and it only started occurring recently. Can you provide some more info on that?
With which EasyBuild version did this start popping up?

Resolving patch:
diff --git a/easybuild/tools/include.py b/easybuild/tools/include.py
--- a/easybuild/tools/include.py
+++ b/easybuild/tools/include.py
@@ -125,10 +125,16 @@ def set_up_eb_package(parent_path, eb_pkg_name, subpkgs=None, pkg_init_body=None
 
 def verify_imports(pymods, pypkg, from_path):
     """Verify that import of specified modules from specified package and expected location works."""
-    saved_modules = sys.modules.copy()
 
     for pymod in pymods:
         pymod_spec = '%s.%s' % (pypkg, pymod)
+
+        # force re-import if the specified modules was already imported;
+        # this is required to ensure that an easyblock that is included via --include-easyblocks-from-pr
+        # gets preference over one that is included via --include-easyblocks
+        if pymod_spec in sys.modules:
+            del sys.modules[pymod_spec]
+
         try:
             pymod = __import__(pymod_spec, fromlist=[pypkg])
         # different types of exceptions may be thrown, not only ImportErrors
@@ -142,10 +148,6 @@ def verify_imports(pymods, pypkg, from_path):
 
         _log.debug("Import of %s from %s verified", pymod_spec, from_path)
 
-    # restore sys.modules to its original state (not only verified modules but also their dependencies)
-    sys.modules.clear()
-    sys.modules.update(saved_modules)
-
 
 def is_software_specific_easyblock(module):
     """Determine whether Python module at specified location is a software-specific easyblock."""
@@ -279,7 +281,7 @@ def include_toolchains(tmpdir, paths):
         sys.modules[tcpkg].__path__.insert(0, os.path.join(toolchains_path, 'easybuild', 'toolchains', subpkg))
 
     # sanity check: verify that included toolchain modules can be imported (from expected location)
-    verify_imports([os.path.splitext(mns)[0] for mns in included_toolchains], 'easybuild.toolchains', tcs_dir)
+    verify_imports([os.path.splitext(tc)[0] for tc in included_toolchains], 'easybuild.toolchains', tcs_dir)
     for subpkg in toolchain_subpkgs:
         pkg = '.'.join(['easybuild', 'toolchains', subpkg])
         loc = os.path.join(tcs_dir, subpkg)


Test patch:
diff --git a/test/framework/include.py b/test/framework/include.py
--- a/test/framework/include.py
+++ b/test/framework/include.py
@@ -156,8 +156,8 @@ def test_include_easyblocks_priority(self):
         self.assertFalse(os.path.samefile(os.path.dirname(os.path.dirname(foo_pyc_path)), test_easyblocks))
         self.assertTrue(os.path.samefile(foo_real_py_path, os.path.join(myeasyblocks, 'foo.py')))
 
-        # check that the included easyblock is not loaded
-        self.assertFalse('easybuild.easyblocks.foo' in sys.modules)
+        # 'undo' import of foo easyblock
+        del sys.modules['easybuild.easyblocks.foo']
 
     def test_include_mns(self):
         """Test include_module_naming_schemes()."""
diff --git a/test/framework/options.py b/test/framework/options.py
--- a/test/framework/options.py
+++ b/test/framework/options.py
@@ -3047,6 +3047,7 @@ def test_xxx_include_easyblocks_from_pr(self):
             return
 
         orig_local_sys_path = sys.path[:]
+
         fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log')
         os.close(fd)
 
@@ -3095,10 +3096,6 @@ def test_xxx_include_easyblocks_from_pr(self):
         del sys.modules['easybuild.easyblocks.generic.cmakemake']
         os.remove(os.path.join(self.test_prefix, 'foo.py'))
         sys.path = orig_local_sys_path
-        import easybuild.easyblocks
-        reload(easybuild.easyblocks)
-        import easybuild.easyblocks.generic
-        reload(easybuild.easyblocks.generic)
 
         # include test cmakemake easyblock
         cmm_txt = '\n'.join([