Repository: reframe-hpc/reframe

Issue:
Relative paths to `--report-file` can result in `FileNotFoundError`
When we run a simple regression test with setting `--report-file` on CLI as `reframe --report-file=test-report.json -c .`, we will end up with following error (pasting only relevant log lines):

```
[  PASSED  ] Ran 0/0 test case(s) from 0 check(s) (0 failure(s), 0 skipped)
[==========] Finished on Wed Apr  5 10:15:13 2023 
Run report saved in 'test-report.json'

ERROR: run session stopped: file not found error: [Errno 2] No such file or directory: ''
Log file(s) saved in '/tmp/rfm-dz27cl1s.log'
```

This is due to the fact that [basedir](https://github.com/reframe-hpc/reframe/blob/6e22addc096cb61640baaa8d9b8851c4975beda3/reframe/frontend/cli.py#L1339-L1344) here is being calculated on relative path of `report_file` and in the current case, `basedir` would be an empty string. Ensuring `report_file` to be an absolute path should fix the issue. This regression comes from this [commit](https://github.com/reframe-hpc/reframe/commit/d01138163e6ab5093827d9807f890435ad629b26)

Discussion:

Patch:
diff --git a/docs/manpage.rst b/docs/manpage.rst
index 2624963b..2730cdf6 100644
--- a/docs/manpage.rst
+++ b/docs/manpage.rst
@@ -367,11 +367,15 @@ Options controlling ReFrame output
    The file where ReFrame will store its report.
 
    The ``FILE`` argument may contain the special placeholder ``{sessionid}``, in which case ReFrame will generate a new report each time it is run by appending a counter to the report file.
+   If the report is generated in the default location (see the :attr:`~config.general.report_file` configuration option), a symlink to the latest report named ``latest.json`` will also be created.
 
    This option can also be set using the :envvar:`RFM_REPORT_FILE` environment variable or the :attr:`~config.general.report_file` general configuration parameter.
 
    .. versionadded:: 3.1
 
+   .. versionadded:: 4.2
+      Symlink to the latest report is now created.
+
 .. option:: --report-junit=FILE
 
    Instruct ReFrame to generate a JUnit XML report in ``FILE``.
diff --git a/reframe/core/runtime.py b/reframe/core/runtime.py
index 5df7f8ba..339d848f 100644
--- a/reframe/core/runtime.py
+++ b/reframe/core/runtime.py
@@ -170,10 +170,23 @@ class RuntimeContext:
         :returns: The value of the option.
 
         .. versionchanged:: 3.11.0
-          Add ``default`` named argument.
+           Add ``default`` named argument.
         '''
         return self._site_config.get(option, default=default)
 
+    def get_default(self, option):
+        '''Get the default value for the option as defined in the configuration
+        schema.
+
+        :arg option: The option whose default value is requested
+        :returns: The default value of the requested option
+        :raises KeyError: if option does not have a default value
+
+        .. versionadded:: 4.2
+        '''
+
+        return self._site_config.schema['defaults'][option]
+
 
 # Global resources for the current host
 _runtime_context = None
diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index dba59e24..1bb2dc80 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -3,7 +3,6 @@
 #
 # SPDX-License-Identifier: BSD-3-Clause
 
-import functools
 import inspect
 import itertools
 import json
@@ -1379,36 +1378,18 @@ def main():
                     json_report['restored_cases'].append(report.case(*c))
 
             report_file = runreport.next_report_filename(report_file)
+            default_loc = os.path.dirname(
+                osext.expandvars(rt.get_default('general/report_file'))
+            )
             try:
-                with open(report_file, 'w') as fp:
-                    if rt.get_option('general/0/compress_report'):
-                        jsonext.dump(json_report, fp)
-                    else:
-                        jsonext.dump(json_report, fp, indent=2)
-                        fp.write('\n')
-
-                printer.info(f'Run report saved in {report_file!r}')
+                runreport.write_report(json_report, report_file,
+                                       rt.get_option(
+                                           'general/0/compress_report'),
+                                       os.path.dirname(report_file) == default_loc)
             except OSError as e:
                 printer.warning(
                     f'failed to generate report in {report_file!r}: {e}'
                 )
-            else:
-                # Add a symlink to the latest report
-                with osext.change_dir(basedir):
-                    link_name = 'latest.json'
-                    create_symlink = functools.partial(
-                        os.symlink, os.path.basename(report_file), link_name
-                    )
-                    if not os.path.exists(link_name):
-                        create_symlink()
-                    else:
-                        if os.path.islink(link_name):
-                            os.remove(link_name)
-                            create_symlink()
-                        else:
-                            printer.warning('could not create a symlink '
-                                            'to the latest report file; '
-                                            'path exists and is not a symlink')
 
             # Generate the junit xml report for this session
             junit_report_file = rt.get_option('general/0/report_junit')
diff --git a/reframe/frontend/runreport.py b/reframe/frontend/runreport.py
index 9550617b..3095f8f1 100644
--- a/reframe/frontend/runreport.py
+++ b/reframe/frontend/runreport.py
@@ -4,6 +4,7 @@
 # SPDX-License-Identifier: BSD-3-Clause
 
 import decimal
+import functools
 import json
 import jsonschema
 import lxml.etree as etree
@@ -13,6 +14,8 @@ import re
 import reframe as rfm
 import reframe.core.exceptions as errors
 import reframe.utility.jsonext as jsonext
+import reframe.utility.osext as osext
+from reframe.core.logging import getlogger
 from reframe.core.warnings import suppress_deprecations
 
 # The schema data version
@@ -179,6 +182,36 @@ def load_report(*filenames):
     return rpt
 
 
+def write_report(report, filename, compress=False, link_to_last=False):
+    with open(filename, 'w') as fp:
+        if compress:
+            jsonext.dump(report, fp)
+        else:
+            jsonext.dump(report, fp, indent=2)
+            fp.write('\n')
+
+        if not link_to_last:
+            return
+
+        # Add a symlink to the latest report
+        basedir = os.path.dirname(filename)
+        with osext.change_dir(basedir):
+            link_name = 'latest.json'
+            create_symlink = functools.partial(
+                os.symlink, os.path.basename(filename), link_name
+            )
+            if not os.path.exists(link_name):
+                create_symlink()
+            else:
+                if os.path.islink(link_name):
+                    os.remove(link_name)
+                    create_symlink()
+                else:
+                    getlogger().warning('could not create a symlink '
+                                        'to the latest report file: '
+                                        'path exists and is not a symlink')
+
+
 def junit_xml_report(json_report):
     '''Generate a JUnit report from a standard ReFrame JSON report.'''
 


Test patch:
diff --git a/test_reframe.py b/test_reframe.py
index 938b86d1..fbeb134a 100755
--- a/test_reframe.py
+++ b/test_reframe.py
@@ -39,7 +39,12 @@ if __name__ == '__main__':
         '--rfm-help', action='help', help='Print this help message and exit.'
     )
     options, rem_args = parser.parse_known_args()
-    test_util.USER_CONFIG_FILE = options.rfm_user_config
+
+    user_config = options.rfm_user_config
+    if user_config is not None:
+        user_config = os.path.abspath(user_config)
+
+    test_util.USER_CONFIG_FILE = user_config
     test_util.USER_SYSTEM = options.rfm_user_system
     test_util.init_runtime()
 
diff --git a/unittests/resources/config/settings.py b/unittests/resources/config/settings.py
index 4e559566..7f82bcd8 100644
--- a/unittests/resources/config/settings.py
+++ b/unittests/resources/config/settings.py
@@ -241,7 +241,7 @@ site_configuration = {
         {
             'name': 'unittest',
             'options': [
-                '-c unittests/resources/checks/hellocheck.py',
+                '-n ^HelloTest$',
                 '-p builtin',
                 '-S local=1'
             ]
diff --git a/unittests/test_cli.py b/unittests/test_cli.py
index 4a1943f9..22d80868 100644
--- a/unittests/test_cli.py
+++ b/unittests/test_cli.py
@@ -4,6 +4,7 @@
 # SPDX-License-Identifier: BSD-3-Clause
 
 import contextlib
+import functools
 import io
 import itertools
 import json
@@ -14,10 +15,16 @@ import sys
 import time
 
 import reframe.core.environments as env
-import reframe.frontend.runreport as runreport
 import reframe.core.logging as logging
 import reframe.core.runtime as rt
+import reframe.frontend.runreport as runreport
+import reframe.utility.osext as osext
 import unittests.utility as test_util
+from reframe import INSTALL_PREFIX
+
+
+# Absolute path relative to the installation path
+norm_path = functools.partial(os.path.join, INSTALL_PREFIX)
 
 
 def run_command_inline(argv, funct, *args, **kwargs):
@@ -29,7 +36,7 @@ def run_command_inline(argv, funct, *args, **kwargs):
 
     captured_stdout = io.StringIO()
     captured_stderr = io.StringIO()
-    print(*sys.argv)
+    print(f'pushd {os.getcwd()}; ', *sys.argv, '; popd')
     with contextlib.redirect_stdout(captured_stdout):
         with contextlib.redirect_stderr(captured_stderr):
             try:
@@ -48,38 +55,36 @@ def run_command_inline(argv, funct, *args, **kwargs):
 
 
 @pytest.fixture
-def perflogdir(tmp_path):
-    dirname = tmp_path / '.rfm-perflogs'
-    yield dirname
-
-
-@pytest.fixture
-def run_reframe(tmp_path, perflogdir, monkeypatch):
+def run_reframe(tmp_path, monkeypatch):
     def _run_reframe(system='generic:default',
-                     checkpath=['unittests/resources/checks/hellocheck.py'],
-                     environs=['builtin'],
+                     checkpath=None,
+                     environs=None,
                      local=True,
                      action='run',
                      more_options=None,
                      mode=None,
-                     config_file='unittests/resources/config/settings.py',
-                     perflogdir=str(perflogdir)):
+                     config_file=None,
+                     perflogdir=None):
         import reframe.frontend.cli as cli
 
-        # We always pass the --report-file option, because we don't want to
-        # pollute the user's home directory
-        argv = ['./bin/reframe', '--prefix', str(tmp_path), '--nocolor',
-                f'--report-file={tmp_path / "report.json"}']
+        argv = [norm_path('bin/reframe'), '--nocolor']
         if mode:
             argv += ['--mode', mode]
 
         if system:
             argv += ['--system', system]
 
-        if config_file:
-            argv += ['-C', config_file]
+        if config_file is None:
+            config_file = norm_path('unittests/resources/config/settings.py')
+
+        if checkpath is None:
+            checkpath = ['unittests/resources/checks/hellocheck.py']
+
+        if environs is None:
+            environs = ['builtin']
 
-        argv += itertools.chain(*(['-c', c] for c in checkpath))
+        argv += ['-C', config_file]
+        argv += itertools.chain(*(['-c', norm_path(c)] for c in checkpath))
         argv += itertools.chain(*(['-p', e] for e in environs))
         if local:
             argv += ['-S', 'local=1']
@@ -110,7 +115,8 @@ def run_reframe(tmp_path, perflogdir, monkeypatch):
         return run_command_inline(argv, cli.main)
 
     monkeypatch.setenv('HOME', str(tmp_path))
-    return _run_reframe
+    with osext.change_dir(tmp_path):
+        yield _run_reframe
 
 
 @pytest.fixture
@@ -148,7 +154,8 @@ def test_check_success(run_reframe, tmp_path, run_action):
 
     logfile = logging.log_files()[0]
     assert os.path.exists(tmp_path / 'output' / logfile)
-    assert os.path.exists(tmp_path / 'report.json')
+    assert os.path.exists(tmp_path / '.reframe' /
+                          'reports' / 'run-report-0.json')
 
 
 def test_check_restore_session_failed(run_reframe, tmp_path):
@@ -157,11 +164,9 @@ def test_check_restore_session_failed(run_reframe, tmp_path):
     )
     returncode, stdout, _ = run_reframe(
         checkpath=[],
-        more_options=[
-            f'--restore-session={tmp_path}/report.json', '--failed'
-        ]
+        more_options=['--restore-session', '--failed']
     )
-    report = runreport.load_report(f'{tmp_path}/report.json')
+    report = runreport.load_report(f'{tmp_path}/.reframe/reports/latest.json')
     assert set(report.slice('name', when=('fail_phase', 'sanity'))) == {'T2'}
     assert set(report.slice('name',
                             when=('fail_phase', 'startup'))) == {'T7', 'T9'}
@@ -179,11 +184,9 @@ def test_check_restore_session_succeeded_test(run_reframe, tmp_path):
     )
     returncode, stdout, _ = run_reframe(
         checkpath=[],
-        more_options=[
-            f'--restore-session={tmp_path}/report.json', '-n', 'T1'
-        ]
+        more_options=['--restore-session', '-n', 'T1']
     )
-    report = runreport.load_report(f'{tmp_path}/report.json')
+    report = runreport.load_report(f'{tmp_path}/.reframe/reports/latest.json')
     assert report['runs'][-1]['num_cases'] == 1
     assert report['runs'][-1]['testcases'][0]['name'] == 'T1'
 
@@ -196,10 +199,8 @@ def test_check_restore_session_check_search_path(run_reframe, tmp_path):
         checkpath=['unittests/resources/checks_unlisted/deps_complex.py']
     )
     returncode, stdout, _ = run_reframe(
-        checkpath=[f'{tmp_path}/foo'],
-        more_options=[
-            f'--restore-session={tmp_path}/report.json', '-n', 'T1', '-R'
-        ],
+        checkpath=[f'foo/'],
+        more_options=['--restore-session', '-n', 'T1', '-R'],
         action='list'
     )
     assert returncode == 0
@@ -218,28 +219,33 @@ def test_check_success_force_local(run_reframe, tmp_path, run_action):
 
 def test_report_file_with_sessionid(run_reframe, tmp_path, run_action):
     returncode, *_ = run_reframe(
-        more_options=[
-            f'--report-file={tmp_path / "rfm-report-{sessionid}.json"}'
-        ],
+        more_options=['--report-file=report-{sessionid}.json'],
         action=run_action
     )
     assert returncode == 0
-    assert os.path.exists(tmp_path / 'rfm-report-0.json')
-    assert os.readlink(tmp_path / 'latest.json') == 'rfm-report-0.json'
+    assert os.path.exists(tmp_path / 'report-0.json')
+
+    # Assert no symlink is created when the `--report-file` is passed
+    assert not os.path.exists(tmp_path / 'latest.json')
+    assert not os.path.exists(tmp_path / '.reframe' /
+                              'reports' / 'latest.json')
 
 
 def test_report_ends_with_newline(run_reframe, tmp_path, run_action):
-    returncode, stdout, _ = run_reframe(
-        more_options=[
-            f'--report-file={tmp_path / "rfm-report.json"}'
-        ],
-        action=run_action
-    )
+    returncode, stdout, _ = run_reframe(action=run_action)
     assert returncode == 0
-    with open(tmp_path / 'rfm-report.json') as fp:
+    with open(tmp_path / '.reframe' / 'reports' / 'run-report-0.json') as fp:
         assert fp.read()[-1] == '\n'
 
-    assert os.readlink(tmp_path / 'latest.json') == 'rfm-report.json'
+
+def test_report_file_symlink_latest(run_reframe, tmp_path, run_action):
+    returncode, stdout, _ = run_reframe(action=run_action)
+    assert returncode == 0
+    assert os.path.exists(tmp_path / '.reframe' /
+                          'reports' / 'run-report-0.json')
+
+    symlink = tmp_path / '.reframe' / 'reports' / 'latest.json'
+    assert os.readlink(symlink) == 'run-report-0.json'
 
 
 def test_check_submit_success(run_reframe, remote_exec_ctx, run_action):
@@ -373,8 +379,7 @@ def test_checkpath_symlink(run_reframe, tmp_path):
     assert num_checks_in_checkdir == num_checks_default
 
 
-def test_performance_check_failure(run_reframe, tmp_path,
-                                   perflogdir, run_action):
+def test_performance_check_failure(run_reframe, tmp_path, run_action):
     returncode, stdout, stderr = run_reframe(
         checkpath=['unittests/resources/checks/frontend_checks.py'],
         more_options=['-n', 'PerformanceFailureCheck'],
@@ -395,7 +400,7 @@ def test_performance_check_failure(run_reframe, tmp_path,
         'builtin' / 'PerformanceFailureCheck'
     )
     if run_action == 'run':
-        assert os.path.exists(perflogdir / 'generic' /
+        assert os.path.exists(tmp_path / 'perflogs' / 'generic' /
                               'default' / 'PerformanceFailureCheck.log')
 
 
@@ -511,17 +516,13 @@ def test_same_output_stage_dir(run_reframe, tmp_path):
 
 def test_execution_modes(run_reframe, run_action):
     returncode, stdout, stderr = run_reframe(
-        checkpath=[],
-        environs=[],
-        local=False,
-        mode='unittest',
-        action=run_action
+        mode='unittest', action=run_action
     )
     assert 'Traceback' not in stdout
     assert 'Traceback' not in stderr
     assert 'FAILED' not in stdout
     assert 'PASSED' in stdout
-    assert 'Ran 2/2 test case' in stdout
+    assert 'Ran 1/1 test case' in stdout
 
 
 def test_invalid_mode_warning(run_reframe):
@@ -776,7 +777,7 @@ def test_quiesce_with_check(run_reframe):
 
 
 def test_load_user_modules(run_reframe, user_exec_ctx):
-    with rt.module_use('unittests/modules'):
+    with rt.module_use(norm_path('unittests/modules')):
         returncode, stdout, stderr = run_reframe(
             more_options=['-m testmod_foo'],
             action='list'
@@ -798,7 +799,7 @@ def test_unload_module(run_reframe, user_exec_ctx):
     if not test_util.has_sane_modules_system():
         pytest.skip('no modules system found')
 
-    with rt.module_use('unittests/modules'):
+    with rt.module_use(norm_path('unittests/modules')):
         ms.load_module('testmod_foo')
         returncode, stdout, stderr = run_reframe(
             more_options=['-u testmod_foo'],
@@ -817,7 +818,7 @@ def test_unuse_module_path(run_reframe, user_exec_ctx, run_action):
     if not test_util.has_sane_modules_system():
         pytest.skip('no modules system found')
 
-    module_path = 'unittests/modules'
+    module_path = norm_path('unittests/modules')
     ms.searchpath_add(module_path)
     returncode, stdout, stderr = run_reframe(
         more_options=[f'--module-path=-{module_path}', '--module=testmod_foo'],
@@ -834,7 +835,7 @@ def test_use_module_path(run_reframe, user_exec_ctx, run_action):
     if not test_util.has_sane_modules_system():
         pytest.skip('no modules system found')
 
-    module_path = 'unittests/modules'
+    module_path = norm_path('unittests/modules')
     returncode, stdout, stderr = run_reframe(
         more_options=[f'--module-path=+{module_path}', '--module=testmod_foo'],
         config_file=test_util.USER_CONFIG_FILE, action=run_action,
@@ -850,7 +851,7 @@ def test_overwrite_module_path(run_reframe, user_exec_ctx, run_action):
     if not test_util.has_sane_modules_system():
         pytest.skip('no modules system found')
 
-    module_path = 'unittests/modules'
+    module_path = norm_path('unittests/modules')
     with contextlib.suppress(KeyError):
         module_path += f':{os.environ["MODULEPATH"]}'