Repository: easybuilders/easybuild-framework

Issue:
eb --skip --rebuild isn't filtering out already built modules correctly
Doing a --skip --rebuild on Python-2.7.16 will rebuild things like:
more-itertools wcwidth MarkupSafe Jinja2 Pygments Babel Sphinx Click

The modulename conversioni of those lowercase(), sed /-/_/g, is not taken into account during skip filtering.


Discussion:

Resolving patch:
diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -1473,17 +1473,18 @@ def skip_extensions(self):
             raise EasyBuildError("Skipping of extensions, but no exts_filter set in easyconfig")
 
         res = []
-        for ext in self.exts:
-            cmd, stdin = resolve_exts_filter_template(exts_filter, ext)
+        for ext_inst in self.ext_instances:
+            cmd, stdin = resolve_exts_filter_template(exts_filter, ext_inst)
             (cmdstdouterr, ec) = run_cmd(cmd, log_all=False, log_ok=False, simple=False, inp=stdin, regexp=False)
             self.log.info("exts_filter result %s %s", cmdstdouterr, ec)
             if ec:
-                self.log.info("Not skipping %s" % ext['name'])
-                self.log.debug("exit code: %s, stdout/err: %s" % (ec, cmdstdouterr))
-                res.append(ext)
+                self.log.info("Not skipping %s", ext_inst.name)
+                self.log.debug("exit code: %s, stdout/err: %s", ec, cmdstdouterr)
+                res.append(ext_inst)
             else:
-                self.log.info("Skipping %s" % ext['name'])
-        self.exts = res
+                self.log.info("Skipping %s", ext_inst.name)
+
+        self.ext_instances = res
 
     #
     # MISCELLANEOUS UTILITY FUNCTIONS
@@ -2077,9 +2078,6 @@ def extensions_step(self, fetch=False):
 
         self.exts_all = self.exts[:]  # retain a copy of all extensions, regardless of filtering/skipping
 
-        if self.skip:
-            self.skip_extensions()
-
         # actually install extensions
         self.log.debug("Installing extensions")
         exts_defaultclass = self.cfg['exts_defaultclass']
@@ -2100,14 +2098,8 @@ def extensions_step(self, fetch=False):
 
         # get class instances for all extensions
         self.ext_instances = []
-        exts_cnt = len(self.exts)
-        for idx, ext in enumerate(self.exts):
-            self.log.debug("Starting extension %s" % ext['name'])
-            tup = (ext['name'], ext.get('version', ''), idx+1, exts_cnt)
-            print_msg("installing extension %s %s (%d/%d)..." % tup, silent=self.silent)
-
-            # always go back to original work dir to avoid running stuff from a dir that no longer exists
-            change_dir(self.orig_workdir)
+        for ext in self.exts:
+            self.log.debug("Creating class instance for extension %s...", ext['name'])
 
             cls, inst = None, None
             class_name = encode_class_name(ext['name'])
@@ -2119,11 +2111,11 @@ def extensions_step(self, fetch=False):
                 # with a similar name (e.g., Perl Extension 'GO' vs 'Go' for which 'EB_Go' is available)
                 cls = get_easyblock_class(None, name=ext['name'], error_on_failed_import=False,
                                           error_on_missing_easyblock=False)
-                self.log.debug("Obtained class %s for extension %s" % (cls, ext['name']))
+                self.log.debug("Obtained class %s for extension %s", cls, ext['name'])
                 if cls is not None:
                     inst = cls(self, ext)
             except (ImportError, NameError) as err:
-                self.log.debug("Failed to use extension-specific class for extension %s: %s" % (ext['name'], err))
+                self.log.debug("Failed to use extension-specific class for extension %s: %s", ext['name'], err)
 
             # alternative attempt: use class specified in class map (if any)
             if inst is None and ext['name'] in exts_classmap:
@@ -2141,7 +2133,7 @@ def extensions_step(self, fetch=False):
             if inst is None:
                 try:
                     cls = get_class_for(default_class_modpath, default_class)
-                    self.log.debug("Obtained class %s for installing extension %s" % (cls, ext['name']))
+                    self.log.debug("Obtained class %s for installing extension %s", cls, ext['name'])
                     inst = cls(self, ext)
                     self.log.debug("Installing extension %s with default class %s (from %s)",
                                    ext['name'], default_class, default_class_modpath)
@@ -2149,7 +2141,23 @@ def extensions_step(self, fetch=False):
                     raise EasyBuildError("Also failed to use default class %s from %s for extension %s: %s, giving up",
                                          default_class, default_class_modpath, ext['name'], err)
             else:
-                self.log.debug("Installing extension %s with class %s (from %s)" % (ext['name'], class_name, mod_path))
+                self.log.debug("Installing extension %s with class %s (from %s)", ext['name'], class_name, mod_path)
+
+            self.ext_instances.append(inst)
+
+        if self.skip:
+            self.skip_extensions()
+
+        exts_cnt = len(self.exts)
+        for idx, (ext, ext_instance) in enumerate(zip(self.exts, self.ext_instances)):
+
+            self.log.debug("Starting extension %s" % ext['name'])
+
+            # always go back to original work dir to avoid running stuff from a dir that no longer exists
+            change_dir(self.orig_workdir)
+
+            tup = (ext['name'], ext.get('version', ''), idx+1, exts_cnt)
+            print_msg("installing extension %s %s (%d/%d)..." % tup, silent=self.silent)
 
             if self.dry_run:
                 tup = (ext['name'], ext.get('version', ''), cls.__name__)
@@ -2165,18 +2173,15 @@ def extensions_step(self, fetch=False):
             else:
                 # don't reload modules for toolchain, there is no need since they will be loaded already;
                 # the (fake) module for the parent software gets loaded before installing extensions
-                inst.toolchain.prepare(onlymod=self.cfg['onlytcmod'], silent=True, loadmod=False,
+                ext_instance.toolchain.prepare(onlymod=self.cfg['onlytcmod'], silent=True, loadmod=False,
                                        rpath_filter_dirs=self.rpath_filter_dirs)
 
             # real work
-            inst.prerun()
-            txt = inst.run()
+            ext_instance.prerun()
+            txt = ext_instance.run()
             if txt:
                 self.module_extra_extensions += txt
-            inst.postrun()
-
-            # append so we can make us of it later (in sanity_check_step)
-            self.ext_instances.append(inst)
+            ext_instance.postrun()
 
         # cleanup (unload fake module, remove fake module dir)
         if fake_mod_data:


Test patch:
diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py
--- a/test/framework/easyblock.py
+++ b/test/framework/easyblock.py
@@ -833,11 +833,11 @@ def test_skip_extensions_step(self):
             toolchain = SYSTEM
             exts_list = [
                 "ext1",
-                ("ext2", "42", {"source_tmpl": "dummy.tgz"}),
+                ("EXT-2", "42", {"source_tmpl": "dummy.tgz"}),
                 ("ext3", "1.1", {"source_tmpl": "dummy.tgz", "modulename": "real_ext"}),
             ]
             exts_filter = ("\
-                if [ %(ext_name)s == 'ext2' ] && [ %(ext_version)s == '42' ] && [[ %(src)s == *dummy.tgz ]];\
+                if [ %(ext_name)s == 'ext_2' ] && [ %(ext_version)s == '42' ] && [[ %(src)s == *dummy.tgz ]];\
                     then exit 0;\
                 elif [ %(ext_name)s == 'real_ext' ]; then exit 0;\
                 else exit 1; fi", "")
@@ -850,11 +850,14 @@ def test_skip_extensions_step(self):
         eb.installdir = config.install_path()
         eb.skip = True
         eb.extensions_step(fetch=True)
-        # 'ext1' should be in eb.exts
-        eb_exts = [y for x in eb.exts for y in x.values()]
+        # 'ext1' should be in eb.ext_instances
+        eb_exts = [x.name for x in eb.ext_instances]
         self.assertTrue('ext1' in eb_exts)
-        # 'ext2' should not
-        self.assertFalse('ext2' in eb_exts)
+        # 'EXT-2' should not
+        self.assertFalse('EXT-2' in eb_exts)
+        self.assertFalse('EXT_2' in eb_exts)
+        self.assertFalse('ext-2' in eb_exts)
+        self.assertFalse('ext_2' in eb_exts)
         # 'ext3' should not
         self.assertFalse('ext3' in eb_exts)
 
diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/dummyextension.py b/test/framework/sandbox/easybuild/easyblocks/generic/dummyextension.py
--- a/test/framework/sandbox/easybuild/easyblocks/generic/dummyextension.py
+++ b/test/framework/sandbox/easybuild/easyblocks/generic/dummyextension.py
@@ -32,3 +32,11 @@
 
 class DummyExtension(ExtensionEasyBlock):
     """Support for building/installing dummy extensions."""
+
+    def __init__(self, *args, **kwargs):
+
+        super(DummyExtension, self).__init__(*args, **kwargs)
+
+        # use lowercase name as default value for expected module name, and replace '-' with '_'
+        if 'modulename' not in self.options:
+            self.options['modulename'] = self.name.lower().replace('-', '_')