Repository: cocotb/cocotb

Issue:
Backwards compat `TESTCASE`
`TESTCASE` is now a regex, but that might cause more than an intended number of tests to run if you name your tests something like `test_you` and `test_your_mama` and specify `TESTCASE` as `test_you` expecting only `test_you` to run (it will now also match `test_your_mama`). 

We can instead offer backwards compatability on `TESTCASE` and provide a new environment variable for filtering. The new environment variable for filtering will be a single regular experession. `TESTCASE` will work as it did previously by turning the arguments into a regular expression. This could be done in `_initialize_testbench`.


Discussion:

Resolving patch:
diff --git a/src/cocotb/__init__.py b/src/cocotb/__init__.py
--- a/src/cocotb/__init__.py
+++ b/src/cocotb/__init__.py
@@ -225,39 +225,14 @@ def _initialise_testbench_(argv_):
         f"Running tests with cocotb v{__version__} from {os.path.dirname(__file__)}"
     )
 
-    # Create the base handle type
-
     _process_plusargs()
     _process_packages()
-
-    # Seed the Python random number generator to make this repeatable
     _setup_random_seed()
-
-    # Setup DUT object
     _setup_root_handle()
-
     _start_user_coverage()
+    _setup_regression_manager()
 
-    global regression_manager
-    regression_manager = RegressionManager()
-
-    # discover tests
-    module_str = os.getenv("MODULE", "").strip()
-    if not module_str:
-        raise RuntimeError(
-            "Environment variable MODULE, which defines the module(s) to execute, is not defined or empty."
-        )
-    modules = [s.strip() for s in module_str.split(",") if s.strip()]
-    regression_manager.setup_pytest_assertion_rewriting()
-    regression_manager.discover_tests(*modules)
-
-    # filter tests
-    test_str = os.getenv("TESTCASE", "").strip()
-    if test_str:
-        filters = [s.strip() for s in test_str.split(",") if s.strip()]
-        regression_manager.add_filters(*filters)
-        regression_manager.set_mode(RegressionMode.TESTCASE)
-
+    # setup global scheduler system
     global _scheduler
     _scheduler = Scheduler(test_complete_cb=regression_manager._test_complete)
 
@@ -432,3 +407,36 @@ def _setup_root_handle() -> None:
 
     global top
     top = cocotb.handle.SimHandle(handle)
+
+
+def _setup_regression_manager() -> None:
+    global regression_manager
+    regression_manager = RegressionManager()
+
+    # discover tests
+    module_str = os.getenv("MODULE", "").strip()
+    if not module_str:
+        raise RuntimeError(
+            "Environment variable MODULE, which defines the module(s) to execute, is not defined or empty."
+        )
+    modules = [s.strip() for s in module_str.split(",") if s.strip()]
+    regression_manager.setup_pytest_assertion_rewriting()
+    regression_manager.discover_tests(*modules)
+
+    # filter tests
+    testcase_str = os.getenv("TESTCASE", "").strip()
+    test_filter_str = os.getenv("COCOTB_TEST_FILTER", "").strip()
+    if testcase_str and test_filter_str:
+        raise RuntimeError("Specify only one of TESTCASE or COCOTB_TEST_FILTER")
+    elif testcase_str:
+        warnings.warn(
+            "TESTCASE is deprecated in favor of COCOTB_TEST_FILTER",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        filters = [f"{s.strip()}$" for s in testcase_str.split(",") if s.strip()]
+        regression_manager.add_filters(*filters)
+        regression_manager.set_mode(RegressionMode.TESTCASE)
+    elif test_filter_str:
+        regression_manager.add_filters(test_filter_str)
+        regression_manager.set_mode(RegressionMode.TESTCASE)
diff --git a/src/cocotb_tools/runner.py b/src/cocotb_tools/runner.py
--- a/src/cocotb_tools/runner.py
+++ b/src/cocotb_tools/runner.py
@@ -295,6 +295,7 @@ def test(
         verbose: bool = False,
         timescale: Optional[Timescale] = None,
         log_file: Optional[PathLike] = None,
+        test_filter: Optional[str] = None,
     ) -> Path:
         """Run the tests.
 
@@ -326,6 +327,8 @@ def test(
                 Typically Tcl commands for simulators that support them.
             timescale: Tuple containing time unit and time precision for simulation.
             log_file: File to write the test log to.
+            test_filter: Regular expression which matches test names.
+                Only matched tests are run if this argument if given.
 
         Returns:
             The absolute location of the results XML file which can be
@@ -377,6 +380,9 @@ def test(
             else:
                 self.env["TESTCASE"] = ",".join(testcase)
 
+        if test_filter is not None:
+            self.env["COCOTB_TEST_FILTER"] = test_filter
+
         if seed is not None:
             self.env["RANDOM_SEED"] = str(seed)
 


Test patch:
diff --git a/tests/test_cases/issue_253/Makefile b/tests/test_cases/issue_253/Makefile
--- a/tests/test_cases/issue_253/Makefile
+++ b/tests/test_cases/issue_253/Makefile
@@ -45,19 +45,19 @@ $(COCOTB_RESULTS_FILE):
 
 notset_top_level: $(SIM_BUILD)/sim.vvp $(CUSTOM_SIM_DEPS)
 	MODULE=$(MODULE) \
-	TESTCASE=issue_253_notset TOPLEVEL= \
+	COCOTB_TEST_FILTER=issue_253_notset TOPLEVEL= \
 	vvp -M $(shell cocotb-config --lib-dir) -m $(shell cocotb-config --lib-name vpi icarus) $(SIM_BUILD)/sim.vvp $(SIM_ARGS) $(EXTRA_ARGS) $(PLUSARGS)
 	mkdir -p $@_result && mv results.xml $@_result/
 
 empty_top_level: $(SIM_BUILD)/sim.vvp $(CUSTOM_SIM_DEPS)
 	MODULE=$(MODULE) \
-	TESTCASE=issue_253_empty TOPLEVEL="" \
+	COCOTB_TEST_FILTER=issue_253_empty TOPLEVEL="" \
 	vvp -M $(shell cocotb-config --lib-dir) -m $(shell cocotb-config --lib-name vpi icarus) $(SIM_BUILD)/sim.vvp $(SIM_ARGS) $(EXTRA_ARGS) $(PLUSARGS)
 	mkdir -p $@_result && mv results.xml $@_result/
 
 no_top_level: $(SIM_BUILD)/sim.vvp $(CUSTOM_SIM_DEPS)
 	MODULE=$(MODULE) \
-	TESTCASE=issue_253_none \
+	COCOTB_TEST_FILTER=issue_253_none \
 	vvp -M $(shell cocotb-config --lib-dir) -m $(shell cocotb-config --lib-name vpi icarus) $(SIM_BUILD)/sim.vvp $(SIM_ARGS) $(EXTRA_ARGS) $(PLUSARGS)
 	mkdir -p $@_result && mv results.xml $@_result/
 
diff --git a/tests/test_cases/test_seed/Makefile b/tests/test_cases/test_seed/Makefile
--- a/tests/test_cases/test_seed/Makefile
+++ b/tests/test_cases/test_seed/Makefile
@@ -5,6 +5,6 @@
 run:
 	$(RM) number
 	$(MAKE) sim MODULE=test_other,test_seed RANDOM_SEED=1234
-	$(MAKE) sim MODULE=test_seed TESTCASE=test_reproducibility RANDOM_SEED=1234
+	$(MAKE) sim MODULE=test_seed COCOTB_TEST_FILTER=test_reproducibility RANDOM_SEED=1234
 
 include ../../designs/sample_module/Makefile
diff --git a/tests/test_cases/test_select_testcase/Makefile b/tests/test_cases/test_select_testcase/Makefile
--- a/tests/test_cases/test_select_testcase/Makefile
+++ b/tests/test_cases/test_select_testcase/Makefile
@@ -8,3 +8,8 @@ include ../../designs/sample_module/Makefile
 
 MODULE := x_tests,y_tests,y_tests_again
 TESTCASE := y_test
+
+# override PYTHONWARNINGS to prevent DeprecationWarning causing an error with use of TESTCASE
+ifdef PYTHONWARNINGS
+override PYTHONWARNINGS = error,ignore::DeprecationWarning:site,always::FutureWarning:cocotb._scheduler,ignore::DeprecationWarning:attr,ignore::DeprecationWarning:cocotb
+endif
diff --git a/tests/test_cases/test_select_testcase/x_tests.py b/tests/test_cases/test_select_testcase/x_tests.py
--- a/tests/test_cases/test_select_testcase/x_tests.py
+++ b/tests/test_cases/test_select_testcase/x_tests.py
@@ -1,3 +1,7 @@
+# Copyright cocotb contributors
+# Licensed under the Revised BSD License, see LICENSE for details.
+# SPDX-License-Identifier: BSD-3-Clause
+
 import cocotb
 
 
diff --git a/tests/test_cases/test_select_testcase/y_tests.py b/tests/test_cases/test_select_testcase/y_tests.py
--- a/tests/test_cases/test_select_testcase/y_tests.py
+++ b/tests/test_cases/test_select_testcase/y_tests.py
@@ -1,3 +1,7 @@
+# Copyright cocotb contributors
+# Licensed under the Revised BSD License, see LICENSE for details.
+# SPDX-License-Identifier: BSD-3-Clause
+
 import cocotb
 
 
diff --git a/tests/test_cases/test_select_testcase/y_tests_again.py b/tests/test_cases/test_select_testcase/y_tests_again.py
--- a/tests/test_cases/test_select_testcase/y_tests_again.py
+++ b/tests/test_cases/test_select_testcase/y_tests_again.py
@@ -1,6 +1,15 @@
+# Copyright cocotb contributors
+# Licensed under the Revised BSD License, see LICENSE for details.
+# SPDX-License-Identifier: BSD-3-Clause
+
 import cocotb
 
 
 @cocotb.test()
 async def y_test(dut):
     pass
+
+
+@cocotb.test()
+async def y_test_with_additional(_):
+    assert False, "TESTCASE shouldn't match this test"
diff --git a/tests/test_cases/test_select_testcase_error/Makefile b/tests/test_cases/test_select_testcase_error/Makefile
--- a/tests/test_cases/test_select_testcase_error/Makefile
+++ b/tests/test_cases/test_select_testcase_error/Makefile
@@ -10,3 +10,8 @@ TESTCASE := y_test
 # TESTCASE filtering out all tests results in a warning
 
 include ../../designs/sample_module/Makefile
+
+# override PYTHONWARNINGS to prevent DeprecationWarning causing an error with use of TESTCASE
+ifdef PYTHONWARNINGS
+override PYTHONWARNINGS = error,ignore::DeprecationWarning:site,always::FutureWarning:cocotb._scheduler,ignore::DeprecationWarning:attr,ignore::DeprecationWarning:cocotb
+endif
diff --git a/tests/test_cases/test_select_testcase_error/x_tests.py b/tests/test_cases/test_select_testcase_error/x_tests.py
--- a/tests/test_cases/test_select_testcase_error/x_tests.py
+++ b/tests/test_cases/test_select_testcase_error/x_tests.py
@@ -1,3 +1,7 @@
+# Copyright cocotb contributors
+# Licensed under the Revised BSD License, see LICENSE for details.
+# SPDX-License-Identifier: BSD-3-Clause
+
 import cocotb
 
 
diff --git a/tests/test_cases/test_skipped/Makefile b/tests/test_cases/test_skipped/Makefile
--- a/tests/test_cases/test_skipped/Makefile
+++ b/tests/test_cases/test_skipped/Makefile
@@ -16,7 +16,7 @@ clean::
 override: clean all
 	 @test -f $(SKIPPED_TEST_FILE) || (echo "ERROR: skip=True test was not ran!" >&2 && exit 1)
 
-# Set TESTCASE; run test_skipped even though skip=True is set.
-TESTCASE = test_skipped
+# Set COCOTB_TEST_FILTER; run test_skipped even though skip=True is set.
+COCOTB_TEST_FILTER = test_skipped
 
 MODULE = test_skipped
diff --git a/tests/test_cases/test_test_filter/Makefile b/tests/test_cases/test_test_filter/Makefile
new file mode 100644
--- /dev/null
+++ b/tests/test_cases/test_test_filter/Makefile
@@ -0,0 +1,10 @@
+# Copyright cocotb contributors
+# Licensed under the Revised BSD License, see LICENSE for details.
+# SPDX-License-Identifier: BSD-3-Clause
+
+# select only y_tests from all MODULEs
+
+include ../../designs/sample_module/Makefile
+
+MODULE := x_tests
+export COCOTB_TEST_FILTER := y_test
diff --git a/tests/test_cases/test_test_filter/x_tests.py b/tests/test_cases/test_test_filter/x_tests.py
new file mode 100644
--- /dev/null
+++ b/tests/test_cases/test_test_filter/x_tests.py
@@ -0,0 +1,24 @@
+# Copyright cocotb contributors
+# Licensed under the Revised BSD License, see LICENSE for details.
+# SPDX-License-Identifier: BSD-3-Clause
+
+import cocotb
+
+
+@cocotb.test
+async def x_test(_):
+    assert False, "Should not match this test"
+
+
+y_test_with_suffix_ran = False
+
+
+@cocotb.test
+async def y_test_with_suffix(_):
+    global y_test_with_suffix_ran
+    y_test_with_suffix_ran = True
+
+
+@cocotb.test
+async def y_test(_):
+    assert y_test_with_suffix_ran