Repository: easybuilders/easybuild-framework

Issue:
Checksums ignored for `sources` of `exts_list`
When an extension uses the `sources` key the checksum is ignored.

See https://github.com/easybuilders/easybuild-framework/blob/8291a1b97f18db7c83df189e23e251ce5f19efae/easybuild/framework/easyblock.py#L551-L566

A bit below it can be seen how checksums are checked when not using `sources`


Discussion:

Resolving patch:
diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py
--- a/easybuild/framework/easyblock.py
+++ b/easybuild/framework/easyblock.py
@@ -502,6 +502,8 @@ def fetch_extension_sources(self, skip_checksums=False):
         if self.dry_run:
             self.dry_run_msg("\nList of sources/patches for extensions:")
 
+        force_download = build_option('force_download') in [FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_SOURCES]
+
         for ext in exts_list:
             if (isinstance(ext, list) or isinstance(ext, tuple)) and ext:
 
@@ -539,18 +541,17 @@ def fetch_extension_sources(self, skip_checksums=False):
                     # resolve templates in extension options
                     ext_options = resolve_template(ext_options, template_values)
 
+                    source_urls = ext_options.get('source_urls', [])
                     checksums = ext_options.get('checksums', [])
 
-                    # use default template for name of source file if none is specified
-                    default_source_tmpl = resolve_template('%(name)s-%(version)s.tar.gz', template_values)
-                    fn = ext_options.get('source_tmpl', default_source_tmpl)
-
                     if ext_options.get('nosource', None):
-                        exts_sources.append(ext_src)
+                        self.log.debug("No sources for extension %s, as indicated by 'nosource'", ext_name)
 
                     elif ext_options.get('sources', None):
                         sources = ext_options['sources']
 
+                        # only a single source file is supported for extensions currently,
+                        # see https://github.com/easybuilders/easybuild-framework/issues/3463
                         if isinstance(sources, list):
                             if len(sources) == 1:
                                 source = sources[0]
@@ -560,67 +561,90 @@ def fetch_extension_sources(self, skip_checksums=False):
                         else:
                             source = sources
 
+                        # always pass source spec as dict value to fetch_source method,
+                        # mostly so we can inject stuff like source URLs
+                        if isinstance(source, string_type):
+                            source = {'filename': source}
+                        elif not isinstance(source, dict):
+                            raise EasyBuildError("Incorrect value type for source of extension %s: %s",
+                                                 ext_name, source)
+
+                        # if no custom source URLs are specified in sources spec,
+                        # inject the ones specified for this extension
+                        if 'source_urls' not in source:
+                            source['source_urls'] = source_urls
+
                         src = self.fetch_source(source, checksums, extension=True)
-                        # Copy 'path' entry to 'src' for use with extensions
-                        ext_src.update({'src': src['path']})
-                        exts_sources.append(ext_src)
-                    else:
-                        source_urls = ext_options.get('source_urls', [])
-                        force_download = build_option('force_download') in [FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_SOURCES]
 
-                        src_fn = self.obtain_file(fn, extension=True, urls=source_urls, force_download=force_download)
+                        # copy 'path' entry to 'src' for use with extensions
+                        ext_src.update({'src': src['path']})
 
-                        if src_fn:
-                            ext_src.update({'src': src_fn})
+                    else:
+                        # use default template for name of source file if none is specified
+                        default_source_tmpl = resolve_template('%(name)s-%(version)s.tar.gz', template_values)
+
+                        # if no sources are specified via 'sources', fall back to 'source_tmpl'
+                        src_fn = ext_options.get('source_tmpl', default_source_tmpl)
+                        src_path = self.obtain_file(src_fn, extension=True, urls=source_urls,
+                                                    force_download=force_download)
+                        if src_path:
+                            ext_src.update({'src': src_path})
+                        else:
+                            raise EasyBuildError("Source for extension %s not found.", ext)
 
-                            if not skip_checksums:
-                                # report both MD5 and SHA256 checksums, since both are valid default checksum types
+                    # verify checksum for extension sources
+                    if 'src' in ext_src and not skip_checksums:
+                        src_path = ext_src['src']
+                        src_fn = os.path.basename(src_path)
+
+                        # report both MD5 and SHA256 checksums, since both are valid default checksum types
+                        for checksum_type in (CHECKSUM_TYPE_MD5, CHECKSUM_TYPE_SHA256):
+                            src_checksum = compute_checksum(src_path, checksum_type=checksum_type)
+                            self.log.info("%s checksum for %s: %s", checksum_type, src_path, src_checksum)
+
+                        # verify checksum (if provided)
+                        self.log.debug('Verifying checksums for extension source...')
+                        fn_checksum = self.get_checksum_for(checksums, index=0)
+                        if verify_checksum(src_path, fn_checksum):
+                            self.log.info('Checksum for extension source %s verified', src_fn)
+                        elif build_option('ignore_checksums'):
+                            print_warning("Ignoring failing checksum verification for %s" % src_fn)
+                        else:
+                            raise EasyBuildError('Checksum verification for extension source %s failed', src_fn)
+
+                    # locate extension patches (if any), and verify checksums
+                    ext_patches = self.fetch_patches(patch_specs=ext_options.get('patches', []), extension=True)
+                    if ext_patches:
+                        self.log.debug('Found patches for extension %s: %s', ext_name, ext_patches)
+                        ext_src.update({'patches': ext_patches})
+
+                        if not skip_checksums:
+                            for patch in ext_patches:
+                                patch = patch['path']
+                                # report both MD5 and SHA256 checksums,
+                                # since both are valid default checksum types
                                 for checksum_type in (CHECKSUM_TYPE_MD5, CHECKSUM_TYPE_SHA256):
-                                    src_checksum = compute_checksum(src_fn, checksum_type=checksum_type)
-                                    self.log.info("%s checksum for %s: %s", checksum_type, src_fn, src_checksum)
-
-                                # verify checksum (if provided)
-                                self.log.debug('Verifying checksums for extension source...')
-                                fn_checksum = self.get_checksum_for(checksums, index=0)
-                                if verify_checksum(src_fn, fn_checksum):
-                                    self.log.info('Checksum for extension source %s verified', fn)
+                                    checksum = compute_checksum(patch, checksum_type=checksum_type)
+                                    self.log.info("%s checksum for %s: %s", checksum_type, patch, checksum)
+
+                            # verify checksum (if provided)
+                            self.log.debug('Verifying checksums for extension patches...')
+                            for idx, patch in enumerate(ext_patches):
+                                patch = patch['path']
+                                patch_fn = os.path.basename(patch)
+
+                                checksum = self.get_checksum_for(checksums[1:], index=idx)
+                                if verify_checksum(patch, checksum):
+                                    self.log.info('Checksum for extension patch %s verified', patch_fn)
                                 elif build_option('ignore_checksums'):
-                                    print_warning("Ignoring failing checksum verification for %s" % fn)
+                                    print_warning("Ignoring failing checksum verification for %s" % patch_fn)
                                 else:
-                                    raise EasyBuildError('Checksum verification for extension source %s failed', fn)
-
-                            ext_patches = self.fetch_patches(patch_specs=ext_options.get('patches', []), extension=True)
-                            if ext_patches:
-                                self.log.debug('Found patches for extension %s: %s' % (ext_name, ext_patches))
-                                ext_src.update({'patches': ext_patches})
-
-                                if not skip_checksums:
-                                    for patch in ext_patches:
-                                        patch = patch['path']
-                                        # report both MD5 and SHA256 checksums,
-                                        # since both are valid default checksum types
-                                        for checksum_type in (CHECKSUM_TYPE_MD5, CHECKSUM_TYPE_SHA256):
-                                            checksum = compute_checksum(patch, checksum_type=checksum_type)
-                                            self.log.info("%s checksum for %s: %s", checksum_type, patch, checksum)
-
-                                    # verify checksum (if provided)
-                                    self.log.debug('Verifying checksums for extension patches...')
-                                    for idx, patch in enumerate(ext_patches):
-                                        patch = patch['path']
-                                        checksum = self.get_checksum_for(checksums[1:], index=idx)
-                                        if verify_checksum(patch, checksum):
-                                            self.log.info('Checksum for extension patch %s verified', patch)
-                                        elif build_option('ignore_checksums'):
-                                            print_warning("Ignoring failing checksum verification for %s" % patch)
-                                        else:
-                                            raise EasyBuildError('Checksum for extension patch %s failed', patch)
-                            else:
-                                self.log.debug('No patches found for extension %s.' % ext_name)
-
-                            exts_sources.append(ext_src)
+                                    raise EasyBuildError("Checksum verification for extension patch %s failed",
+                                                         patch_fn)
+                    else:
+                        self.log.debug('No patches found for extension %s.' % ext_name)
 
-                        else:
-                            raise EasyBuildError("Source for extension %s not found.", ext)
+                    exts_sources.append(ext_src)
 
             elif isinstance(ext, string_type):
                 exts_sources.append({'name': ext})


Test patch:
diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py
--- a/test/framework/toy_build.py
+++ b/test/framework/toy_build.py
@@ -51,8 +51,8 @@
 from easybuild.tools.build_log import EasyBuildError
 from easybuild.tools.config import get_module_syntax, get_repositorypath
 from easybuild.tools.environment import modify_env
-from easybuild.tools.filetools import adjust_permissions, change_dir, mkdir, read_file, remove_dir, remove_file
-from easybuild.tools.filetools import which, write_file
+from easybuild.tools.filetools import adjust_permissions, change_dir, copy_file, mkdir
+from easybuild.tools.filetools import read_file, remove_dir, remove_file, which, write_file
 from easybuild.tools.module_generator import ModuleGeneratorTcl
 from easybuild.tools.modules import Lmod
 from easybuild.tools.py2vs3 import reload, string_type
@@ -1196,45 +1196,123 @@ def test_toy_extension_patches(self):
 
         self.test_toy_build(ec_file=test_ec)
 
-    def test_toy_extension_sources_single_item_list(self):
+    def test_toy_extension_sources(self):
         """Test install toy that includes extensions with 'sources' spec (as single-item list)."""
-        test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
+        topdir = os.path.dirname(os.path.abspath(__file__))
+        test_ecs = os.path.join(topdir, 'easyconfigs', 'test_ecs')
         toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb')
         toy_ec_txt = read_file(toy_ec)
 
         test_ec = os.path.join(self.test_prefix, 'test.eb')
 
-        # test use of single-element list in 'sources' with just the filename
-        test_ec_txt = '\n'.join([
-            toy_ec_txt,
-            'exts_list = [',
-            '   ("bar", "0.0", {',
-            '       "sources": ["bar-%(version)s.tar.gz"],',
-            '   }),',
-            ']',
-        ])
-        write_file(test_ec, test_ec_txt)
-        self.test_toy_build(ec_file=test_ec)
+        bar_sources_specs = [
+            '["bar-%(version)s.tar.gz"]',  # single-element list
+            '"bar-%(version)s.tar.gz"',  # string value
+        ]
 
-    def test_toy_extension_sources_str(self):
-        """Test install toy that includes extensions with 'sources' spec (as string value)."""
-        test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
-        toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb')
-        toy_ec_txt = read_file(toy_ec)
+        for bar_sources_spec in bar_sources_specs:
 
-        test_ec = os.path.join(self.test_prefix, 'test.eb')
+            # test use of single-element list in 'sources' with just the filename
+            test_ec_txt = '\n'.join([
+                toy_ec_txt,
+                'exts_list = [',
+                '   ("bar", "0.0", {',
+                '       "sources": %s,' % bar_sources_spec,
+                '   }),',
+                ']',
+            ])
+            write_file(test_ec, test_ec_txt)
+            self.test_toy_build(ec_file=test_ec)
 
-        # test use of single-element list in 'sources' with just the filename
-        test_ec_txt = '\n'.join([
-            toy_ec_txt,
-            'exts_list = [',
-            '   ("bar", "0.0", {',
-            '       "sources": "bar-%(version)s.tar.gz",',
-            '   }),',
-            ']',
-        ])
-        write_file(test_ec, test_ec_txt)
-        self.test_toy_build(ec_file=test_ec)
+            # copy bar-0.0.tar.gz to <tmpdir>/bar-0.0-local.tar.gz, to be used below
+            test_source_path = os.path.join(self.test_prefix, 'sources')
+            toy_ext_sources = os.path.join(topdir, 'sandbox', 'sources', 'toy', 'extensions')
+
+            bar_source = os.path.join(toy_ext_sources, 'bar-0.0.tar.gz')
+            copy_file(bar_source, os.path.join(test_source_path, 'bar-0.0-local.tar.gz'))
+
+            bar_patch = os.path.join(toy_ext_sources, 'bar-0.0_fix-silly-typo-in-printf-statement.patch')
+            copy_file(bar_patch, os.path.join(self.test_prefix, 'bar-0.0_fix-local.patch'))
+
+            # verify that source_urls and patches are picked up and taken into account
+            # when 'sources' is used to specify extension sources
+
+            bar_sources_spec = bar_sources_spec.replace('bar-%(version)s.tar.gz', 'bar-0.0-local.tar.gz')
+
+            test_ec_txt = '\n'.join([
+                toy_ec_txt,
+                'exts_list = [',
+                '   ("bar", "0.0", {',
+                '       "source_urls": ["file://%s"],' % test_source_path,
+                '       "sources": %s,' % bar_sources_spec,
+                '       "patches": ["bar-%(version)s_fix-local.patch"],',
+                '   }),',
+                ']',
+            ])
+            write_file(test_ec, test_ec_txt)
+            self.test_toy_build(ec_file=test_ec, raise_error=True)
+
+            # check that checksums are picked up and verified
+            test_ec_txt = '\n'.join([
+                toy_ec_txt,
+                'exts_list = [',
+                '   ("bar", "0.0", {',
+                '       "source_urls": ["file://%s"],' % test_source_path,
+                '       "sources": %s,' % bar_sources_spec,
+                '       "patches": ["bar-%(version)s_fix-local.patch"],',
+                # note: purposely incorrect (SHA256) checksums! (to check if checksum verification works)
+                '       "checksums": [',
+                '           "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff",',
+                '           "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff",',
+                '       ],',
+                '   }),',
+                ']',
+            ])
+            write_file(test_ec, test_ec_txt)
+
+            error_pattern = r"Checksum verification for extension source bar-0.0-local.tar.gz failed"
+            self.assertErrorRegex(EasyBuildError, error_pattern, self.test_toy_build, ec_file=test_ec,
+                                  raise_error=True, verbose=False)
+
+            # test again with correct checksum for bar-0.0.tar.gz, but faulty checksum for patch file
+            test_ec_txt = '\n'.join([
+                toy_ec_txt,
+                'exts_list = [',
+                '   ("bar", "0.0", {',
+                '       "source_urls": ["file://%s"],' % test_source_path,
+                '       "sources": %s,' % bar_sources_spec,
+                '       "patches": ["bar-%(version)s_fix-local.patch"],',
+                '       "checksums": [',
+                '           "f3676716b610545a4e8035087f5be0a0248adee0abb3930d3edb76d498ae91e7",',
+                # note: purposely incorrect checksum for patch!
+                '           "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff",',
+                '       ],',
+                '   }),',
+                ']',
+            ])
+            write_file(test_ec, test_ec_txt)
+
+            error_pattern = r"Checksum verification for extension patch bar-0.0_fix-local.patch failed"
+            self.assertErrorRegex(EasyBuildError, error_pattern, self.test_toy_build, ec_file=test_ec,
+                                  raise_error=True, verbose=False)
+
+            # test again with correct checksums
+            test_ec_txt = '\n'.join([
+                toy_ec_txt,
+                'exts_list = [',
+                '   ("bar", "0.0", {',
+                '       "source_urls": ["file://%s"],' % test_source_path,
+                '       "sources": %s,' % bar_sources_spec,
+                '       "patches": ["bar-%(version)s_fix-local.patch"],',
+                '       "checksums": [',
+                '           "f3676716b610545a4e8035087f5be0a0248adee0abb3930d3edb76d498ae91e7",',
+                '           "84db53592e882b5af077976257f9c7537ed971cb2059003fd4faa05d02cae0ab",',
+                '       ],',
+                '   }),',
+                ']',
+            ])
+            write_file(test_ec, test_ec_txt)
+            self.test_toy_build(ec_file=test_ec, raise_error=True)
 
     def test_toy_extension_sources_git_config(self):
         """Test install toy that includes extensions with 'sources' spec including 'git_config'."""