Repository: reframe-hpc/reframe

Issue:
Filtering tests based on the number of nodes
Today we can filter individual tests based on their name using a regex, but the number of nodes is not always part of the test's name, for instance when `num_nodes` is not parameterized. It would be great to be able to specify a maximum (and possibly minimum) number of nodes to use and have ReFrame automatically ignore tests that do not match the constraint.

I discussed with @vkarak and he pointed out related features in https://github.com/reframe-hpc/reframe/issues/2665 and https://github.com/reframe-hpc/reframe/issues/2674, but those features would still require knowledge of which configurations are supported by each test, and thus might require setting different parameters for each test.

Whereas the request here would be less fine grained but would not require a priori knowledge about the existing tests, for instance it would work with tests using any of those statements:
```python
num_nodes = 1
num_nodes = 16
num_nodes = parameter(range(16))
num_nodes = parameter([10, 40, 80])
num_nodes = parameter([2, 4, 8, 16, 32])
num_nodes = int(os.getenv("TEST_NNODES", "10"))
```



Comments:

Patch:
diff --git a/docs/manpage.rst b/docs/manpage.rst
index ee2348cd..3b458538 100644
--- a/docs/manpage.rst
+++ b/docs/manpage.rst
@@ -68,6 +68,21 @@ This happens recursively so that if test ``T1`` depends on ``T2`` and ``T2`` dep
    The value of this attribute is not required to be non-zero for GPU tests.
    Tests may or may not make use of it.
 
+   .. deprecated:: 4.4
+
+      Please use ``-E 'not num_gpus_per_node'`` instead.
+
+.. option:: -E, --filter-expr=EXPR
+
+   Select only tests that satisfy the given expression.
+
+   The expression ``EXPR`` can be any valid Python expression on the test variables or parameters.
+   For example, ``-E num_tasks > 10`` will select all tests, whose :attr:`~reframe.core.pipeline.RegressionTest.num_tasks` exceeds ``10``.
+   You may use any test variable in expression, even user-defined.
+   Multiple variables can also be included such as ``-E num_tasks >= my_param``, where ``my_param`` is user-defined parameter.
+
+   .. versionadded:: 4.4
+
 .. option:: --failed
 
    Select only the failed test cases for a previous run.
@@ -77,6 +92,7 @@ This happens recursively so that if test ``T1`` depends on ``T2`` and ``T2`` dep
 
    .. versionadded:: 3.4
 
+
 .. option:: --gpu-only
 
    Select tests that can run on GPUs.
@@ -84,6 +100,10 @@ This happens recursively so that if test ``T1`` depends on ``T2`` and ``T2`` dep
    These are all tests with :attr:`num_gpus_per_node` greater than zero.
    This option and :option:`--cpu-only` are mutually exclusive.
 
+   .. deprecated:: 4.4
+
+      Please use ``-E num_gpus_per_node`` instead.
+
 .. option:: --maintainer=MAINTAINER
 
    Filter tests by maintainer.
@@ -101,6 +121,7 @@ This happens recursively so that if test ``T1`` depends on ``T2`` and ``T2`` dep
       The ``MAINTAINER`` pattern is matched anywhere in the maintainer's name and not at its beginning.
       If you want to match at the beginning of the name, you should prepend ``^``.
 
+
 .. option:: -n, --name=NAME
 
    Filter tests by name.
diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index 7b4ea826..8c59a680 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -356,6 +356,10 @@ def main():
         metavar='PATTERN', default=[],
         help='Exclude checks whose name matches PATTERN'
     )
+    select_options.add_argument(
+        '-E', '--filter-expr', action='store', metavar='EXPR',
+        help='Select checks that satisfy the expression EXPR'
+    )
 
     # Action options
     action_options.add_argument(
@@ -1048,6 +1052,16 @@ def main():
             f'Filtering test cases(s) by tags: {len(testcases)} remaining'
         )
 
+        if options.filter_expr:
+            testcases = filter(filters.validates(options.filter_expr),
+                               testcases)
+
+            testcases = list(testcases)
+            printer.verbose(
+                f'Filtering test cases(s) by {options.filter_expr}: '
+                f'{len(testcases)} remaining'
+            )
+
         # Filter test cases by maintainers
         for maint in options.maintainers:
             testcases = filter(filters.have_maintainer(maint), testcases)
@@ -1059,8 +1073,12 @@ def main():
             sys.exit(1)
 
         if options.gpu_only:
+            printer.warning('the `--gpu-only` option is deprecated; '
+                            'please use `-E num_gpus_per_node` instead')
             testcases = filter(filters.have_gpu_only(), testcases)
         elif options.cpu_only:
+            printer.warning('the `--cpu-only` option is deprecated; '
+                            'please use `-E "not num_gpus_per_node"` instead')
             testcases = filter(filters.have_cpu_only(), testcases)
 
         testcases = list(testcases)
diff --git a/reframe/frontend/filters.py b/reframe/frontend/filters.py
index f6e80741..2bcc1052 100644
--- a/reframe/frontend/filters.py
+++ b/reframe/frontend/filters.py
@@ -109,16 +109,18 @@ def have_maintainer(patt):
 
 
 def have_gpu_only():
-    def _fn(case):
-        # NOTE: This takes into account num_gpus_per_node being None
-        return case.check.num_gpus_per_node
-
-    return _fn
+    return validates('num_gpus_per_node')
 
 
 def have_cpu_only():
+    return validates('not num_gpus_per_node')
+
+
+def validates(expr):
     def _fn(case):
-        # NOTE: This takes into account num_gpus_per_node being None
-        return not case.check.num_gpus_per_node
+        try:
+            return eval(expr, None, case.check.__dict__)
+        except Exception as err:
+            raise ReframeError(f'invalid expression `{expr}`') from err
 
     return _fn


Test patch:
diff --git a/unittests/test_cli.py b/unittests/test_cli.py
index 14ba5fc5..d5d8f148 100644
--- a/unittests/test_cli.py
+++ b/unittests/test_cli.py
@@ -698,6 +698,42 @@ def test_filtering_exclude_hash(run_reframe):
     assert returncode == 0
 
 
+def test_filtering_cpu_only(run_reframe):
+    returncode, stdout, stderr = run_reframe(
+        checkpath=['unittests/resources/checks/hellocheck.py'],
+        action='list',
+        more_options=['--cpu-only']
+    )
+    assert 'Traceback' not in stdout
+    assert 'Traceback' not in stderr
+    assert 'Found 2 check(s)' in stdout
+    assert returncode == 0
+
+
+def test_filtering_gpu_only(run_reframe):
+    returncode, stdout, stderr = run_reframe(
+        checkpath=['unittests/resources/checks/hellocheck.py'],
+        action='list',
+        more_options=['--gpu-only']
+    )
+    assert 'Traceback' not in stdout
+    assert 'Traceback' not in stderr
+    assert 'Found 0 check(s)' in stdout
+    assert returncode == 0
+
+
+def test_filtering_by_expr(run_reframe):
+    returncode, stdout, stderr = run_reframe(
+        checkpath=['unittests/resources/checks/hellocheck.py'],
+        action='list',
+        more_options=['-E num_tasks==1']
+    )
+    assert 'Traceback' not in stdout
+    assert 'Traceback' not in stderr
+    assert 'Found 2 check(s)' in stdout
+    assert returncode == 0
+
+
 def test_show_config_all(run_reframe):
     # Just make sure that this option does not make the frontend crash
     returncode, stdout, stderr = run_reframe(
diff --git a/unittests/test_filters.py b/unittests/test_filters.py
index 1a06c2f8..30e91b23 100644
--- a/unittests/test_filters.py
+++ b/unittests/test_filters.py
@@ -11,6 +11,7 @@ import reframe.frontend.executors as executors
 import reframe.frontend.filters as filters
 import reframe.utility.sanity as sn
 import unittests.utility as test_util
+from reframe.core.exceptions import ReframeError
 
 
 def count_checks(filter_fn, checks):
@@ -140,3 +141,30 @@ def test_invalid_regex(sample_cases):
 
     with pytest.raises(errors.ReframeError):
         count_checks(filters.have_tag('*foo'), sample_cases).evaluate()
+
+
+def test_validates_expr(sample_cases, sample_param_cases):
+    validates = filters.validates
+    assert count_checks(validates('"a" in tags'), sample_cases) == 2
+    assert count_checks(validates('num_gpus_per_node == 1'), sample_cases) == 2
+    assert count_checks(validates('p > 5'), sample_param_cases) == 5
+    assert count_checks(validates('p > 5 or p < 1'), sample_param_cases) == 6
+    assert count_checks(validates('num_tasks in tags'), sample_cases) == 0
+
+
+def test_validates_expr_invalid(sample_cases):
+    validates = filters.validates
+
+    # undefined variables
+    with pytest.raises(ReframeError):
+        assert count_checks(validates('foo == 3'), sample_cases)
+
+    # invalid syntax
+    with pytest.raises(ReframeError):
+        assert count_checks(validates('num_tasks = 2'), sample_cases)
+
+    with pytest.raises(ReframeError):
+        assert count_checks(validates('import os'), sample_cases)
+
+    with pytest.raises(ReframeError):
+        assert count_checks(validates('"foo" i tags'), sample_cases)