Repository: easybuilders/easybuild-framework

Issue:
test report not generated if there are missing OS dependencies
If there are missing OS dependencies then the test report is not generated.

```
$ eb --from-pr 16454 -Tur
/rds/projects/2017/branfosj-rse/easybuild/venv-dev-EL8/lib64/python3.6/site-packages/secretstorage/util.py:23: CryptographyDeprecationWarning: Python 3.6 is no longer supported by the Python core team. Therefore, support for it is deprecated in cryptography and will be removed in a future release.
  from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
== Temporary log file in case of crash /dev/shm/branfosj/tmp-up-EL8/eb-6zrtd8aw/easybuild-46c2o47d.log
  >> download succeeded: https://github.com/easybuilders/easybuild-easyconfigs/archive/develop.tar.gz
  >> running command:
        [started at: 2022-10-21 09:42:40]
        [working dir: /dev/shm/branfosj/tmp-up-EL8/eb-6zrtd8aw/tmpy4jopged/easybuilders]
        [output logged in /dev/shm/branfosj/tmp-up-EL8/eb-6zrtd8aw/easybuild-run_cmd-fcarxe72.log]
        tar xzf /dev/shm/branfosj/tmp-up-EL8/eb-6zrtd8aw/tmpy4jopged/easybuilders/develop.tar.gz
  >> command completed: exit 0, ran in < 1s
  >> download succeeded: https://github.com/easybuilders/easybuild-easyconfigs/pull/16454.diff
ERROR: Failed to process easyconfig /dev/shm/branfosj/tmp-up-EL8/eb-6zrtd8aw/files_pr16454/b/bsddb3/bsddb3-6.2.9-GCCcore-11.3.0.eb: One or more OS dependencies were not found: [('libdb-dev', 'libdb-devel')]
```


Discussion:

Resolving patch:
diff --git a/easybuild/main.py b/easybuild/main.py
--- a/easybuild/main.py
+++ b/easybuild/main.py
@@ -374,8 +374,20 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session
             sys.exit(31)  # exit -> 3x1t -> 31
 
     # read easyconfig files
-    validate = not options.inject_checksums and not options.inject_checksums_to_json
-    easyconfigs, generated_ecs = parse_easyconfigs(paths, validate=validate)
+    try:
+        validate = not options.inject_checksums and not options.inject_checksums_to_json
+        easyconfigs, generated_ecs = parse_easyconfigs(paths, validate=validate)
+    except Exception as err:
+        # Catch any exception in easyconfig parsing, so we can generate a test report if required
+        if options.dump_test_report or options.upload_test_report:
+            # dump/upload overall test report
+            fail_msg = "Failed during parsing of the easyconfigs, so no ecs were built"
+            test_report_msg = overall_test_report([], len(paths), False, fail_msg, init_session_state,
+                                                  ec_parse_error=err)
+            if test_report_msg is not None:
+                _log.info(test_report_msg)
+
+        raise err
 
     # handle --check-contrib & --check-style options
     if run_contrib_style_checks([ec['ec'] for ec in easyconfigs], options.check_contrib, options.check_style):


Test patch:
diff --git a/easybuild/tools/testing.py b/easybuild/tools/testing.py
--- a/easybuild/tools/testing.py
+++ b/easybuild/tools/testing.py
@@ -138,8 +138,18 @@ def session_state():
     }
 
 
-def create_test_report(msg, ecs_with_res, init_session_state, pr_nrs=None, gist_log=False, easyblock_pr_nrs=None):
-    """Create test report for easyconfigs PR, in Markdown format."""
+def create_test_report(msg, ecs_with_res, init_session_state, pr_nrs=None, gist_log=False, easyblock_pr_nrs=None,
+                       ec_parse_error=None):
+    """
+    Create test report for easyconfigs PR, in Markdown format.
+    :param msg: message to be included in test report
+    :param ecs_with_res: processed easyconfigs with build result (success/failure)
+    :param init_session_state: initial session state info to include in test report
+    :param pr_nrs: list of ints for the PRs in the easyconfigs repository
+    :param gist_log: boolean for whether to create a gist from the log file(s)
+    :param easyblock_pr_nrs:  list of ints for the PRs in the easyblocks repository
+    :param ec_parse_error: error message for easyconfig parse failure
+    """
 
     github_user = build_option('github_user')
     pr_target_account = build_option('pr_target_account')
@@ -170,42 +180,52 @@ def create_test_report(msg, ecs_with_res, init_session_state, pr_nrs=None, gist_
         "",
     ])
 
-    build_overview = ["#### Overview of tested easyconfigs (in order)"]
-    for (ec, ec_res) in ecs_with_res:
-        test_log = ''
-        if ec_res.get('success', False):
-            test_result = 'SUCCESS'
-        else:
-            # compose test result string
-            test_result = 'FAIL '
-            if 'err' in ec_res:
-                if isinstance(ec_res['err'], EasyBuildError):
-                    test_result += '(build issue)'
-                else:
-                    test_result += '(unhandled exception: %s)' % ec_res['err']
-                    test_result += ec_res['traceback']
+    if ec_parse_error:
+        build_overview = [str(ec_parse_error)]
+        test_report.extend([
+            "#### Error",
+            "```",
+            str(ec_parse_error),
+            "```",
+            "",
+        ])
+    else:
+        build_overview = ["#### Overview of tested easyconfigs (in order)"]
+        for (ec, ec_res) in ecs_with_res:
+            test_log = ''
+            if ec_res.get('success', False):
+                test_result = 'SUCCESS'
             else:
-                test_result += '(unknown cause, not an exception?!)'
+                # compose test result string
+                test_result = 'FAIL '
+                if 'err' in ec_res:
+                    if isinstance(ec_res['err'], EasyBuildError):
+                        test_result += '(build issue)'
+                    else:
+                        test_result += '(unhandled exception: %s)' % ec_res['err']
+                        test_result += ec_res['traceback']
+                else:
+                    test_result += '(unknown cause, not an exception?!)'
 
-            # create gist for log file (if desired and available)
-            if gist_log and 'log_file' in ec_res:
-                logtxt = read_file(ec_res['log_file'])
-                partial_log_txt = '\n'.join(logtxt.split('\n')[-500:])
-                descr = "(partial) EasyBuild log for failed build of %s" % ec['spec']
+                # create gist for log file (if desired and available)
+                if gist_log and 'log_file' in ec_res:
+                    logtxt = read_file(ec_res['log_file'])
+                    partial_log_txt = '\n'.join(logtxt.split('\n')[-500:])
+                    descr = "(partial) EasyBuild log for failed build of %s" % ec['spec']
 
-                if pr_nrs:
-                    descr += " (PR(s) #%s)" % ', #'.join(str(x) for x in pr_nrs)
+                    if pr_nrs:
+                        descr += " (PR(s) #%s)" % ', #'.join(str(x) for x in pr_nrs)
 
-                if easyblock_pr_nrs:
-                    descr += " (easyblock PR(s) #%s)" % ', #'.join(str(x) for x in easyblock_pr_nrs)
+                    if easyblock_pr_nrs:
+                        descr += " (easyblock PR(s) #%s)" % ', #'.join(str(x) for x in easyblock_pr_nrs)
 
-                fn = '%s_partial.log' % os.path.basename(ec['spec'])[:-3]
-                gist_url = create_gist(partial_log_txt, fn, descr=descr, github_user=github_user)
-                test_log = "(partial log available at %s)" % gist_url
+                    fn = '%s_partial.log' % os.path.basename(ec['spec'])[:-3]
+                    gist_url = create_gist(partial_log_txt, fn, descr=descr, github_user=github_user)
+                    test_log = "(partial log available at %s)" % gist_url
 
-        build_overview.append(" * **%s** _%s_ %s" % (test_result, os.path.basename(ec['spec']), test_log))
-    build_overview.append("")
-    test_report.extend(build_overview)
+            build_overview.append(" * **%s** _%s_ %s" % (test_result, os.path.basename(ec['spec']), test_log))
+        build_overview.append("")
+        test_report.extend(build_overview)
 
     time_format = "%a, %d %b %Y %H:%M:%S +0000 (UTC)"
     start_time = strftime(time_format, init_session_state['time'])
@@ -344,7 +364,7 @@ def post_pr_test_report(pr_nrs, repo_type, test_report, msg, init_session_state,
     return msg
 
 
-def overall_test_report(ecs_with_res, orig_cnt, success, msg, init_session_state):
+def overall_test_report(ecs_with_res, orig_cnt, success, msg, init_session_state, ec_parse_error=None):
     """
     Upload/dump overall test report
     :param ecs_with_res: processed easyconfigs with build result (success/failure)
@@ -352,6 +372,7 @@ def overall_test_report(ecs_with_res, orig_cnt, success, msg, init_session_state
     :param success: boolean indicating whether all builds were successful
     :param msg: message to be included in test report
     :param init_session_state: initial session state info to include in test report
+    :param ec_parse_error: error message for easyconfig parse failure
     """
     dump_path = build_option('dump_test_report')
 
@@ -371,7 +392,7 @@ def overall_test_report(ecs_with_res, orig_cnt, success, msg, init_session_state
         msg = msg + " (%d easyconfigs in total)" % orig_cnt
 
         test_report = create_test_report(msg, ecs_with_res, init_session_state, pr_nrs=pr_nrs, gist_log=True,
-                                         easyblock_pr_nrs=easyblock_pr_nrs)
+                                         easyblock_pr_nrs=easyblock_pr_nrs, ec_parse_error=ec_parse_error)
         if pr_nrs:
             # upload test report to gist and issue a comment in the PR(s) to notify
             txt = post_pr_test_report(pr_nrs, GITHUB_EASYCONFIGS_REPO, test_report, msg, init_session_state,
@@ -386,7 +407,7 @@ def overall_test_report(ecs_with_res, orig_cnt, success, msg, init_session_state
             gist_url = upload_test_report_as_gist(test_report['full'])
             txt = "Test report uploaded to %s" % gist_url
     else:
-        test_report = create_test_report(msg, ecs_with_res, init_session_state)
+        test_report = create_test_report(msg, ecs_with_res, init_session_state, ec_parse_error=ec_parse_error)
         txt = None
     _log.debug("Test report: %s" % test_report['full'])
 
diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py
--- a/test/framework/toy_build.py
+++ b/test/framework/toy_build.py
@@ -153,7 +153,7 @@ def check_toy(self, installpath, outtxt, version='0.0', versionprefix='', versio
 
     def test_toy_build(self, extra_args=None, ec_file=None, tmpdir=None, verify=True, fails=False, verbose=True,
                        raise_error=False, test_report=None, versionsuffix='', testing=True,
-                       raise_systemexit=False, force=True):
+                       raise_systemexit=False, force=True, test_report_regexs=None):
         """Perform a toy build."""
         if extra_args is None:
             extra_args = []
@@ -198,19 +198,24 @@ def test_toy_build(self, extra_args=None, ec_file=None, tmpdir=None, verify=True
         # make sure full test report was dumped, and contains sensible information
         if test_report is not None:
             self.assertTrue(os.path.exists(test_report))
-            if fails:
-                test_result = 'FAIL'
+            if test_report_regexs:
+                regex_patterns = test_report_regexs
             else:
-                test_result = 'SUCCESS'
-            regex_patterns = [
-                r"Test result[\S\s]*Build succeeded for %d out of 1" % (not fails),
-                r"Overview of tested easyconfig[\S\s]*%s[\S\s]*%s" % (test_result, os.path.basename(ec_file)),
+                if fails:
+                    test_result = 'FAIL'
+                else:
+                    test_result = 'SUCCESS'
+                regex_patterns = [
+                    r"Test result[\S\s]*Build succeeded for %d out of 1" % (not fails),
+                    r"Overview of tested easyconfig[\S\s]*%s[\S\s]*%s" % (test_result, os.path.basename(ec_file)),
+                ]
+            regex_patterns.extend([
                 r"Time info[\S\s]*start:[\S\s]*end:",
                 r"EasyBuild info[\S\s]*framework version:[\S\s]*easyblocks ver[\S\s]*command line[\S\s]*configuration",
                 r"System info[\S\s]*cpu model[\S\s]*os name[\S\s]*os version[\S\s]*python version",
                 r"List of loaded modules",
                 r"Environment",
-            ]
+            ])
             test_report_txt = read_file(test_report)
             for regex_pattern in regex_patterns:
                 regex = re.compile(regex_pattern, re.M)
@@ -3740,6 +3745,39 @@ def test_toy_post_install_patches(self):
         # verify whether patch indeed got applied
         self.assertEqual(toy_readme_txt, "toy 0.0, a toy test program\n")
 
+    def test_toy_unavailable_os_dep(self):
+        """
+        Test unavailable OS dep
+        Existence of OS dependencies is checking during the parsing of the easyconfig.
+        Test here that this problem is caught and a test report generated (#4102).
+        """
+        test_ecs = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs')
+        toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb')
+
+        test_ec_txt = read_file(toy_ec)
+        test_ec_txt += "\nosdependencies = [('package-does-not-exist')]"
+        test_ec = os.path.join(self.test_prefix, 'test.eb')
+        write_file(test_ec, test_ec_txt)
+
+        test_report_fp = os.path.join(self.test_buildpath, 'full_test_report.md')
+
+        self.mock_stderr(True)
+        self.mock_stdout(True)
+        self.test_toy_build(ec_file=test_ec, force=False, raise_error=False, verify=False,
+                            test_report_regexs=[r"One or more OS dependencies were not found"],
+                            test_report=test_report_fp)
+        stdout = self.get_stdout()
+        self.mock_stderr(False)
+        self.mock_stdout(False)
+
+        patterns = [
+            r"Failed to process easyconfig",
+            r"One or more OS dependencies were not found",
+        ]
+        for pattern in patterns:
+            regex = re.compile(pattern, re.M)
+            self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout))
+
 
 def suite():
     """ return all the tests in this file """