Repository: easybuilders/easybuild-framework

Issue:
Sources for extensions are still downloaded with `--module-only`
I was testing `--module-only` with the EESSI stack to build my module tree and my VM ran out of space. I thought it was the CVMFS cache (which was indeed partially to blame) but actually it turned out that the `/tmp` was full and inside I found a sources directory that was populated with lots of extension sources.


Discussion:
This happens because `init_ext_instances` which is triggered via `_sanity_check_step_extensions` calls `fetch_extension_sources` in the `EasyBlock` class (since #3655).

We need those extension instances for the sanity check, but we don't actually need the sources, so we should decouple creating extension instances and downloading sources (so breaking up `fetch_extension_sources`, which now does both).

Resolving patch:
diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -72,9 +72,10 @@
 from easybuild.tools.config import install_path, log_path, package_path, source_paths
 from easybuild.tools.environment import restore_env, sanitize_env
 from easybuild.tools.filetools import CHECKSUM_TYPE_MD5, CHECKSUM_TYPE_SHA256
-from easybuild.tools.filetools import adjust_permissions, apply_patch, back_up_file, change_dir, convert_name
-from easybuild.tools.filetools import compute_checksum, copy_file, check_lock, create_lock, derive_alt_pypi_url
-from easybuild.tools.filetools import diff_files, dir_contains_files, download_file, encode_class_name, extract_file
+from easybuild.tools.filetools import adjust_permissions, apply_patch, back_up_file, change_dir, check_lock
+from easybuild.tools.filetools import compute_checksum, convert_name, copy_file, create_lock, create_patch_info
+from easybuild.tools.filetools import derive_alt_pypi_url, diff_files, dir_contains_files, download_file
+from easybuild.tools.filetools import encode_class_name, extract_file
 from easybuild.tools.filetools import find_backup_name_candidate, get_source_tarball_from_git, is_alt_pypi_url
 from easybuild.tools.filetools import is_binary, is_sha256_checksum, mkdir, move_file, move_logs, read_file, remove_dir
 from easybuild.tools.filetools import remove_file, remove_lock, verify_checksum, weld_paths, write_file, symlink
@@ -372,7 +373,7 @@ def fetch_source(self, source, checksum=None, extension=False):
 
         :param source: source to be found (single dictionary in 'sources' list, or filename)
         :param checksum: checksum corresponding to source
-        :param extension: flag if being called from fetch_extension_sources()
+        :param extension: flag if being called from collect_exts_file_info()
         """
         filename, download_filename, extract_cmd, source_urls, git_config = None, None, None, None, None
 
@@ -461,52 +462,19 @@ def fetch_patches(self, patch_specs=None, extension=False, checksums=None):
         patches = []
         for index, patch_spec in enumerate(patch_specs):
 
-            # check if the patches can be located
-            copy_file = False
-            suff = None
-            level = None
-            if isinstance(patch_spec, (list, tuple)):
-                if not len(patch_spec) == 2:
-                    raise EasyBuildError("Unknown patch specification '%s', only 2-element lists/tuples are supported!",
-                                         str(patch_spec))
-                patch_file = patch_spec[0]
-
-                # this *must* be of typ int, nothing else
-                # no 'isinstance(..., int)', since that would make True/False also acceptable
-                if isinstance(patch_spec[1], int):
-                    level = patch_spec[1]
-                elif isinstance(patch_spec[1], string_type):
-                    # non-patch files are assumed to be files to copy
-                    if not patch_spec[0].endswith('.patch'):
-                        copy_file = True
-                    suff = patch_spec[1]
-                else:
-                    raise EasyBuildError("Wrong patch spec '%s', only int/string are supported as 2nd element",
-                                         str(patch_spec))
-            else:
-                patch_file = patch_spec
+            patch_info = create_patch_info(patch_spec)
 
             force_download = build_option('force_download') in [FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_PATCHES]
-            path = self.obtain_file(patch_file, extension=extension, force_download=force_download)
+            path = self.obtain_file(patch_info['name'], extension=extension, force_download=force_download)
             if path:
-                self.log.debug('File %s found for patch %s' % (path, patch_spec))
-                patchspec = {
-                    'name': patch_file,
-                    'path': path,
-                    'checksum': self.get_checksum_for(checksums, index=index),
-                }
-                if suff:
-                    if copy_file:
-                        patchspec['copy'] = suff
-                    else:
-                        patchspec['sourcepath'] = suff
-                if level is not None:
-                    patchspec['level'] = level
+                self.log.debug('File %s found for patch %s', path, patch_spec)
+                patch_info['path'] = path
+                patch_info['checksum'] = self.get_checksum_for(checksums, index=index)
 
                 if extension:
-                    patches.append(patchspec)
+                    patches.append(patch_info)
                 else:
-                    self.patches.append(patchspec)
+                    self.patches.append(patch_info)
             else:
                 raise EasyBuildError('No file found for patch %s', patch_spec)
 
@@ -514,25 +482,38 @@ def fetch_patches(self, patch_specs=None, extension=False, checksums=None):
             self.log.info("Fetched extension patches: %s", patches)
             return patches
         else:
-            self.log.info("Added patches: %s" % self.patches)
+            self.log.info("Added patches: %s", self.patches)
 
     def fetch_extension_sources(self, skip_checksums=False):
         """
-        Find source file for extensions.
+        Fetch source and patch files for extensions (DEPRECATED, use collect_exts_file_info instead).
+        """
+        depr_msg = "EasyBlock.fetch_extension_sources is deprecated, use EasyBlock.collect_exts_file_info instead"
+        self.log.deprecated(depr_msg, '5.0')
+        return self.collect_exts_file_info(fetch_files=True, verify_checksums=not skip_checksums)
+
+    def collect_exts_file_info(self, fetch_files=True, verify_checksums=True):
+        """
+        Collect information on source and patch files for extensions.
+
+        :param fetch_files: whether or not to fetch files (if False, path to files will be missing from info)
+        :param verify_checksums: whether or not to verify checksums
+        :return: list of dict values, one per extension, with information on source/patch files.
         """
         exts_sources = []
         exts_list = self.cfg.get_ref('exts_list')
 
+        if verify_checksums and not fetch_files:
+            raise EasyBuildError("Can't verify checksums for extension files if they are not being fetched")
+
         if self.dry_run:
             self.dry_run_msg("\nList of sources/patches for extensions:")
 
         force_download = build_option('force_download') in [FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_SOURCES]
 
         for ext in exts_list:
-            if (isinstance(ext, list) or isinstance(ext, tuple)) and ext:
-
+            if isinstance(ext, (list, tuple)) and ext:
                 # expected format: (name, version, options (dict))
-
                 ext_name = ext[0]
                 if len(ext) == 1:
                     exts_sources.append({'name': ext_name})
@@ -602,10 +583,10 @@ def fetch_extension_sources(self, skip_checksums=False):
                         if 'source_urls' not in source:
                             source['source_urls'] = source_urls
 
-                        src = self.fetch_source(source, checksums, extension=True)
-
-                        # copy 'path' entry to 'src' for use with extensions
-                        ext_src.update({'src': src['path']})
+                        if fetch_files:
+                            src = self.fetch_source(source, checksums, extension=True)
+                            # copy 'path' entry to 'src' for use with extensions
+                            ext_src.update({'src': src['path']})
 
                     else:
                         # use default template for name of source file if none is specified
@@ -619,15 +600,16 @@ def fetch_extension_sources(self, skip_checksums=False):
                             error_msg = "source_tmpl value must be a string! (found value of type '%s'): %s"
                             raise EasyBuildError(error_msg, type(src_fn).__name__, src_fn)
 
-                        src_path = self.obtain_file(src_fn, extension=True, urls=source_urls,
-                                                    force_download=force_download)
-                        if src_path:
-                            ext_src.update({'src': src_path})
-                        else:
-                            raise EasyBuildError("Source for extension %s not found.", ext)
+                        if fetch_files:
+                            src_path = self.obtain_file(src_fn, extension=True, urls=source_urls,
+                                                        force_download=force_download)
+                            if src_path:
+                                ext_src.update({'src': src_path})
+                            else:
+                                raise EasyBuildError("Source for extension %s not found.", ext)
 
                     # verify checksum for extension sources
-                    if 'src' in ext_src and not skip_checksums:
+                    if verify_checksums and 'src' in ext_src:
                         src_path = ext_src['src']
                         src_fn = os.path.basename(src_path)
 
@@ -647,12 +629,17 @@ def fetch_extension_sources(self, skip_checksums=False):
                             raise EasyBuildError('Checksum verification for extension source %s failed', src_fn)
 
                     # locate extension patches (if any), and verify checksums
-                    ext_patches = self.fetch_patches(patch_specs=ext_options.get('patches', []), extension=True)
+                    ext_patches = ext_options.get('patches', [])
+                    if fetch_files:
+                        ext_patches = self.fetch_patches(patch_specs=ext_patches, extension=True)
+                    else:
+                        ext_patches = [create_patch_info(p) for p in ext_patches]
+
                     if ext_patches:
                         self.log.debug('Found patches for extension %s: %s', ext_name, ext_patches)
                         ext_src.update({'patches': ext_patches})
 
-                        if not skip_checksums:
+                        if verify_checksums:
                             for patch in ext_patches:
                                 patch = patch['path']
                                 # report both MD5 and SHA256 checksums,
@@ -1995,7 +1982,7 @@ def fetch_step(self, skip_checksums=False):
 
         # fetch extensions
         if self.cfg.get_ref('exts_list'):
-            self.exts = self.fetch_extension_sources(skip_checksums=skip_checksums)
+            self.exts = self.collect_exts_file_info(fetch_files=True, verify_checksums=not skip_checksums)
 
         # create parent dirs in install and modules path already
         # this is required when building in parallel
@@ -2311,8 +2298,11 @@ def init_ext_instances(self):
         self.ext_instances = []
         exts_classmap = self.cfg['exts_classmap']
 
+        # self.exts may already be populated at this point through collect_exts_file_info;
+        # if it's not, we do it lightweight here, by skipping fetching of the files;
+        # information on location of source/patch files will be lacking in that case (but that should be fine)
         if exts_list and not self.exts:
-            self.exts = self.fetch_extension_sources()
+            self.exts = self.collect_exts_file_info(fetch_files=False, verify_checksums=False)
 
         # obtain name and module path for default extention class
         exts_defaultclass = self.cfg['exts_defaultclass']
@@ -2410,7 +2400,7 @@ def extensions_step(self, fetch=False, install=True):
         self.prepare_for_extensions()
 
         if fetch:
-            self.exts = self.fetch_extension_sources()
+            self.exts = self.collect_exts_file_info(fetch_files=True)
 
         self.exts_all = self.exts[:]  # retain a copy of all extensions, regardless of filtering/skipping
 
diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py
--- a/easybuild/tools/filetools.py
+++ b/easybuild/tools/filetools.py
@@ -1409,6 +1409,44 @@ def guess_patch_level(patched_files, parent_dir):
     return patch_level
 
 
+def create_patch_info(patch_spec):
+    """
+    Create info dictionary from specified patch spec.
+    """
+    if isinstance(patch_spec, (list, tuple)):
+        if not len(patch_spec) == 2:
+            error_msg = "Unknown patch specification '%s', only 2-element lists/tuples are supported!"
+            raise EasyBuildError(error_msg, str(patch_spec))
+
+        patch_info = {'name': patch_spec[0]}
+
+        patch_arg = patch_spec[1]
+        # patch level *must* be of type int, nothing else (not True/False!)
+        # note that 'isinstance(..., int)' returns True for True/False values...
+        if isinstance(patch_arg, int) and not isinstance(patch_arg, bool):
+            patch_info['level'] = patch_arg
+
+        # string value as patch argument can be either path where patch should be applied,
+        # or path to where a non-patch file should be copied
+        elif isinstance(patch_arg, string_type):
+            if patch_spec[0].endswith('.patch'):
+                patch_info['sourcepath'] = patch_arg
+            # non-patch files are assumed to be files to copy
+            else:
+                patch_info['copy'] = patch_arg
+        else:
+            raise EasyBuildError("Wrong patch spec '%s', only int/string are supported as 2nd element",
+                                 str(patch_spec))
+
+    elif isinstance(patch_spec, string_type):
+        patch_info = {'name': patch_spec}
+    else:
+        error_msg = "Wrong patch spec, should be string of 2-tuple with patch name + argument: %s"
+        raise EasyBuildError(error_msg, patch_spec)
+
+    return patch_info
+
+
 def apply_patch(patch_file, dest, fn=None, copy=False, level=None, use_git_am=False, use_git=False):
     """
     Apply a patch to source code in directory dest


Test patch:
diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py
--- a/test/framework/easyblock.py
+++ b/test/framework/easyblock.py
@@ -1617,6 +1617,65 @@ def test_fallback_source_url(self):
 
         self.assertTrue(verify_checksum(expected_path, eb.cfg['checksums'][0]))
 
+    def test_collect_exts_file_info(self):
+        """Test collect_exts_file_info method."""
+        testdir = os.path.abspath(os.path.dirname(__file__))
+        toy_sources = os.path.join(testdir, 'sandbox', 'sources', 'toy')
+        toy_ext_sources = os.path.join(toy_sources, 'extensions')
+        toy_ec_file = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-gompi-2018a-test.eb')
+        toy_ec = process_easyconfig(toy_ec_file)[0]
+        toy_eb = EasyBlock(toy_ec['ec'])
+
+        exts_file_info = toy_eb.collect_exts_file_info()
+
+        self.assertTrue(isinstance(exts_file_info, list))
+        self.assertEqual(len(exts_file_info), 4)
+
+        self.assertEqual(exts_file_info[0], {'name': 'ls'})
+
+        self.assertEqual(exts_file_info[1]['name'], 'bar')
+        self.assertEqual(exts_file_info[1]['src'], os.path.join(toy_ext_sources, 'bar-0.0.tar.gz'))
+        bar_patch1 = 'bar-0.0_fix-silly-typo-in-printf-statement.patch'
+        self.assertEqual(exts_file_info[1]['patches'][0]['name'], bar_patch1)
+        self.assertEqual(exts_file_info[1]['patches'][0]['path'], os.path.join(toy_ext_sources, bar_patch1))
+        bar_patch2 = 'bar-0.0_fix-very-silly-typo-in-printf-statement.patch'
+        self.assertEqual(exts_file_info[1]['patches'][1]['name'], bar_patch2)
+        self.assertEqual(exts_file_info[1]['patches'][1]['path'], os.path.join(toy_ext_sources, bar_patch2))
+
+        self.assertEqual(exts_file_info[2]['name'], 'barbar')
+        self.assertEqual(exts_file_info[2]['src'], os.path.join(toy_ext_sources, 'barbar-0.0.tar.gz'))
+        self.assertFalse('patches' in exts_file_info[2])
+
+        self.assertEqual(exts_file_info[3]['name'], 'toy')
+        self.assertEqual(exts_file_info[3]['src'], os.path.join(toy_sources, 'toy-0.0.tar.gz'))
+        self.assertFalse('patches' in exts_file_info[3])
+
+        # location of files is missing when fetch_files is set to False
+        exts_file_info = toy_eb.collect_exts_file_info(fetch_files=False, verify_checksums=False)
+
+        self.assertTrue(isinstance(exts_file_info, list))
+        self.assertEqual(len(exts_file_info), 4)
+
+        self.assertEqual(exts_file_info[0], {'name': 'ls'})
+
+        self.assertEqual(exts_file_info[1]['name'], 'bar')
+        self.assertFalse('src' in exts_file_info[1])
+        self.assertEqual(exts_file_info[1]['patches'][0]['name'], bar_patch1)
+        self.assertFalse('path' in exts_file_info[1]['patches'][0])
+        self.assertEqual(exts_file_info[1]['patches'][1]['name'], bar_patch2)
+        self.assertFalse('path' in exts_file_info[1]['patches'][1])
+
+        self.assertEqual(exts_file_info[2]['name'], 'barbar')
+        self.assertFalse('src' in exts_file_info[2])
+        self.assertFalse('patches' in exts_file_info[2])
+
+        self.assertEqual(exts_file_info[3]['name'], 'toy')
+        self.assertFalse('src' in exts_file_info[3])
+        self.assertFalse('patches' in exts_file_info[3])
+
+        error_msg = "Can't verify checksums for extension files if they are not being fetched"
+        self.assertErrorRegex(EasyBuildError, error_msg, toy_eb.collect_exts_file_info, fetch_files=False)
+
     def test_obtain_file_extension(self):
         """Test use of obtain_file method on an extension."""
 
@@ -2062,9 +2121,16 @@ def test_checksum_step(self):
         error_msg = "Checksum verification for .*/toy-0.0.tar.gz using .* failed"
         self.assertErrorRegex(EasyBuildError, error_msg, eb.checksum_step)
 
-        # also check verification of checksums for extensions, which is part of fetch_extension_sources
+        # also check verification of checksums for extensions, which is part of collect_exts_file_info
         error_msg = "Checksum verification for extension source bar-0.0.tar.gz failed"
+        self.assertErrorRegex(EasyBuildError, error_msg, eb.collect_exts_file_info)
+
+        # also check with deprecated fetch_extension_sources method
+        self.allow_deprecated_behaviour()
+        self.mock_stderr(True)
         self.assertErrorRegex(EasyBuildError, error_msg, eb.fetch_extension_sources)
+        self.mock_stderr(False)
+        self.disallow_deprecated_behaviour()
 
         # if --ignore-checksums is enabled, faulty checksums are reported but otherwise ignored (no error)
         build_options = {
@@ -2084,7 +2150,7 @@ def test_checksum_step(self):
 
         self.mock_stderr(True)
         self.mock_stdout(True)
-        eb.fetch_extension_sources()
+        eb.collect_exts_file_info()
         stderr = self.get_stderr()
         stdout = self.get_stdout()
         self.mock_stderr(False)
@@ -2092,6 +2158,19 @@ def test_checksum_step(self):
         self.assertEqual(stdout, '')
         self.assertEqual(stderr.strip(), "WARNING: Ignoring failing checksum verification for bar-0.0.tar.gz")
 
+        # also check with deprecated fetch_extension_sources method
+        self.allow_deprecated_behaviour()
+        self.mock_stderr(True)
+        self.mock_stdout(True)
+        eb.fetch_extension_sources()
+        stderr = self.get_stderr()
+        stdout = self.get_stdout()
+        self.mock_stderr(False)
+        self.mock_stdout(False)
+        self.assertEqual(stdout, '')
+        self.assertTrue(stderr.strip().endswith("WARNING: Ignoring failing checksum verification for bar-0.0.tar.gz"))
+        self.disallow_deprecated_behaviour()
+
     def test_check_checksums(self):
         """Test for check_checksums_for and check_checksums methods."""
         testdir = os.path.abspath(os.path.dirname(__file__))
diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py
--- a/test/framework/easyconfig.py
+++ b/test/framework/easyconfig.py
@@ -474,7 +474,7 @@ def test_exts_list(self):
         self.prep()
         ec = EasyConfig(self.eb_file)
         eb = EasyBlock(ec)
-        exts_sources = eb.fetch_extension_sources()
+        exts_sources = eb.collect_exts_file_info()
 
         self.assertEqual(len(exts_sources), 2)
         self.assertEqual(exts_sources[0]['name'], 'ext1')
diff --git a/test/framework/filetools.py b/test/framework/filetools.py
--- a/test/framework/filetools.py
+++ b/test/framework/filetools.py
@@ -1510,6 +1510,23 @@ def test_derive_alt_pypi_url(self):
         url = 'https://pypi.python.org/packages/source/n/nosuchpackageonpypiever/nosuchpackageonpypiever-0.0.0.tar.gz'
         self.assertEqual(ft.derive_alt_pypi_url(url), None)
 
+    def test_create_patch_info(self):
+        """Test create_patch_info function."""
+
+        self.assertEqual(ft.create_patch_info('foo.patch'), {'name': 'foo.patch'})
+        self.assertEqual(ft.create_patch_info('foo.txt'), {'name': 'foo.txt'})
+        self.assertEqual(ft.create_patch_info(('foo.patch', 1)), {'name': 'foo.patch', 'level': 1})
+        self.assertEqual(ft.create_patch_info(('foo.patch', 'subdir')), {'name': 'foo.patch', 'sourcepath': 'subdir'})
+        self.assertEqual(ft.create_patch_info(('foo.txt', 'subdir')), {'name': 'foo.txt', 'copy': 'subdir'})
+
+        # faulty input
+        error_msg = "Wrong patch spec"
+        self.assertErrorRegex(EasyBuildError, error_msg, ft.create_patch_info, None)
+        self.assertErrorRegex(EasyBuildError, error_msg, ft.create_patch_info, {'name': 'foo.patch'})
+        self.assertErrorRegex(EasyBuildError, error_msg, ft.create_patch_info, ('foo.patch', [1, 2]))
+        error_msg = "Unknown patch specification"
+        self.assertErrorRegex(EasyBuildError, error_msg, ft.create_patch_info, ('foo.patch', 1, 'subdir'))
+
     def test_apply_patch(self):
         """ Test apply_patch """
         testdir = os.path.dirname(os.path.abspath(__file__))
diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py
--- a/test/framework/toy_build.py
+++ b/test/framework/toy_build.py
@@ -1746,6 +1746,16 @@ def test_module_only_extensions(self):
         # remove module file so we can try --module-only
         remove_file(toy_mod)
 
+        # make sure that sources for extensions can't be found,
+        # they should not be needed when using --module-only
+        # (cfr. https://github.com/easybuilders/easybuild-framework/issues/3849)
+        del os.environ['EASYBUILD_SOURCEPATH']
+
+        # first try normal --module-only, should work fine
+        self.eb_main([test_ec, '--module-only'], do_build=True, raise_error=True)
+        self.assertTrue(os.path.exists(toy_mod))
+        remove_file(toy_mod)
+
         # rename file required for barbar extension, so we can check whether sanity check catches it
         libbarbar = os.path.join(self.test_installpath, 'software', 'toy', '0.0', 'lib', 'libbarbar.a')
         move_file(libbarbar, libbarbar + '.foobar')