Repository: cocotb/cocotb

Issue:
Cleanup @cocotb.test(expect_error=True)
#2107 revealed how `expect_error=True` can mask a problem, because any `Exception` raised by the test will mark it as PASS.

This test passes even though the wrong error is causing the failure:

https://github.com/cocotb/cocotb/blob/89e7bd77b1e9ceb3172b4659e267bc4c42217a0f/tests/test_cases/test_discovery/test_discovery.py#L101-L106

These tests should be cleaned up to use `assert_raises` style or provide the expected `Exception` type.


Discussion:
This test is a duplicate or very similar to what is done in test_handle.py

We agreed to merge/move/remove this in PR #2107 
@garmin-mjames could you please link PR #2107 to this issue, so this is closed when it gets merged? Thanks.
@markusdd There are other uses that should be modified. The test from #2107 is mentioned as the motivating example that showed how using `True` instead of the expected `Exception` type can lead to an inadvertent PASS. The point of this issue is to investigate the uses of `expect_error=True` and be more explicit where it makes sense to do so.
Ah ok, sorry, I didn't interpret this issue to be a general cleanup task.
Passing `True` to `except_error` can also be deprecated.

Resolving patch:
diff --git a/cocotb/decorators.py b/cocotb/decorators.py
--- a/cocotb/decorators.py
+++ b/cocotb/decorators.py
@@ -510,9 +510,8 @@ class test(coroutine, metaclass=_decorator_helper):
         expect_fail (bool, optional):
             Don't mark the result as a failure if the test fails.
 
-        expect_error (bool or exception type or tuple of exception types, optional):
-            If ``True``, consider this test passing if it raises *any* :class:`Exception`, and failing if it does not.
-            If given an exception type or tuple of exception types, catching *only* a listed exception type is considered passing.
+        expect_error (exception type or tuple of exception types, optional):
+            Mark the result as a pass only if one of the exception types is raised in the test.
             This is primarily for cocotb internal regression use for when a simulator error is expected.
 
             Users are encouraged to use the following idiom instead::
@@ -529,6 +528,10 @@ async def my_test(dut):
             .. versionchanged:: 1.3
                 Specific exception types can be expected
 
+            .. deprecated:: 1.5
+                Passing a :class:`bool` value is now deprecated.
+                Pass a specific :class:`Exception` or a tuple of Exceptions instead.
+
         skip (bool, optional):
             Don't execute this test as part of the regression. Test can still be run
             manually by setting :make:var:`TESTCASE`.
@@ -540,7 +543,7 @@ async def my_test(dut):
     _id_count = 0  # used by the RegressionManager to sort tests in definition order
 
     def __init__(self, f, timeout_time=None, timeout_unit="step",
-                 expect_fail=False, expect_error=False,
+                 expect_fail=False, expect_error=(),
                  skip=False, stage=None):
 
         if timeout_unit is None:
@@ -571,6 +574,11 @@ async def f(*args, **kwargs):
         self.timeout_time = timeout_time
         self.timeout_unit = timeout_unit
         self.expect_fail = expect_fail
+        if isinstance(expect_error, bool):
+            warnings.warn(
+                "Passing bool values to `except_error` option of `cocotb.test` is deprecated. "
+                "Pass a specific Exception type instead",
+                DeprecationWarning, stacklevel=2)
         if expect_error is True:
             expect_error = (Exception,)
         elif expect_error is False:


Test patch:
diff --git a/tests/test_cases/issue_120/issue_120.py b/tests/test_cases/issue_120/issue_120.py
--- a/tests/test_cases/issue_120/issue_120.py
+++ b/tests/test_cases/issue_120/issue_120.py
@@ -19,7 +19,7 @@ async def monitor(dut):
 
 
 # Cadence simulators: "Unable set up RisingEdge(...) Trigger" with VHDL (see #1076)
-@cocotb.test(expect_error=cocotb.triggers.TriggerException if cocotb.SIM_NAME.startswith(("xmsim", "ncsim")) and cocotb.LANGUAGE in ["vhdl"] else False)
+@cocotb.test(expect_error=cocotb.triggers.TriggerException if cocotb.SIM_NAME.startswith(("xmsim", "ncsim")) and cocotb.LANGUAGE in ["vhdl"] else ())
 async def issue_120_scheduling(dut):
 
     cocotb.fork(Clock(dut.clk, 10, 'ns').start())
diff --git a/tests/test_cases/issue_142/issue_142.py b/tests/test_cases/issue_142/issue_142.py
--- a/tests/test_cases/issue_142/issue_142.py
+++ b/tests/test_cases/issue_142/issue_142.py
@@ -7,7 +7,7 @@
 
 
 # Cadence simulators: "Unable set up RisingEdge(...) Trigger" with VHDL (see #1076)
-@cocotb.test(expect_error=cocotb.triggers.TriggerException if cocotb.SIM_NAME.startswith(("xmsim", "ncsim")) and cocotb.LANGUAGE in ["vhdl"] else False)
+@cocotb.test(expect_error=cocotb.triggers.TriggerException if cocotb.SIM_NAME.startswith(("xmsim", "ncsim")) and cocotb.LANGUAGE in ["vhdl"] else ())
 async def issue_142_overflow_error(dut):
     """Tranparently convert ints too long to pass
        through the GPI interface natively into BinaryValues"""
diff --git a/tests/test_cases/issue_893/issue_893.py b/tests/test_cases/issue_893/issue_893.py
--- a/tests/test_cases/issue_893/issue_893.py
+++ b/tests/test_cases/issue_893/issue_893.py
@@ -6,7 +6,7 @@ async def coroutine_with_undef():
     new_variable = undefined_variable  # noqa
 
 
-@cocotb.test(expect_error=True)
+@cocotb.test(expect_error=NameError)
 async def fork_erroring_coroutine(dut):
     cocotb.fork(coroutine_with_undef())
     await Timer(10, units='ns')
diff --git a/tests/test_cases/test_cocotb/test_deprecated.py b/tests/test_cases/test_cocotb/test_deprecated.py
--- a/tests/test_cases/test_cocotb/test_deprecated.py
+++ b/tests/test_cases/test_cocotb/test_deprecated.py
@@ -106,3 +106,13 @@ async def test_handle_compat_mapping(dut):
     with assert_deprecated():
         dut.fullname = "myfullname"
     assert dut.fullname == "myfullname"
+
+
+@cocotb.test()
+async def test_expect_error_bool_deprecated(_):
+    async def t():
+        pass
+    with assert_deprecated():
+        cocotb.test(expect_error=True)(t)
+    with assert_deprecated():
+        cocotb.test(expect_error=False)(t)
diff --git a/tests/test_cases/test_cocotb/test_generator_coroutines.py b/tests/test_cases/test_cocotb/test_generator_coroutines.py
--- a/tests/test_cases/test_cocotb/test_generator_coroutines.py
+++ b/tests/test_cases/test_cocotb/test_generator_coroutines.py
@@ -6,7 +6,7 @@
 """
 import cocotb
 from cocotb.triggers import Timer
-from common import clock_gen, _check_traceback
+from common import clock_gen, _check_traceback, assert_raises
 import textwrap
 
 
@@ -91,23 +91,24 @@ def test_yield_list(dut):
 
 
 @cocotb.coroutine
-def syntax_error():
+def erroring_coro():
     yield Timer(100)
     fail  # noqa
 
 
-@cocotb.test(expect_error=True)
-def test_coroutine_syntax_error(dut):
-    """Syntax error in a coroutine that we yield"""
+@cocotb.test()
+def test_coroutine_error(dut):
+    """Error in a coroutine that we yield"""
     yield clock_gen(dut.clk)
-    yield syntax_error()
+    with assert_raises(NameError):
+        yield erroring_coro()
 
 
-@cocotb.test(expect_error=True)
-def test_fork_syntax_error(dut):
-    """Syntax error in a coroutine that we fork"""
+@cocotb.test(expect_error=NameError)
+def test_fork_error(dut):
+    """Error in a coroutine that we fork"""
     yield clock_gen(dut.clk)
-    cocotb.fork(syntax_error())
+    cocotb.fork(erroring_coro())
     yield clock_gen(dut.clk)
 
 
diff --git a/tests/test_cases/test_cocotb/test_handle.py b/tests/test_cases/test_cocotb/test_handle.py
--- a/tests/test_cases/test_cocotb/test_handle.py
+++ b/tests/test_cases/test_cocotb/test_handle.py
@@ -66,7 +66,7 @@ async def test_delayed_assignment_still_errors(dut):
         dut.stream_in_int <= []
 
 
-@cocotb.test(expect_error=cocotb.SIM_NAME in ["Icarus Verilog"])
+@cocotb.test(expect_error=AttributeError if cocotb.SIM_NAME in ["Icarus Verilog"] else ())
 async def test_integer(dut):
     """
     Test access to integers
@@ -83,7 +83,7 @@ async def test_integer(dut):
     assert got_in == got_out, "stream_in_int and stream_out_int should not match"
 
 
-@cocotb.test(expect_error=cocotb.SIM_NAME in ["Icarus Verilog"])
+@cocotb.test(expect_error=AttributeError if cocotb.SIM_NAME in ["Icarus Verilog"] else ())
 async def test_real_assign_double(dut):
     """
     Assign a random floating point value, read it back from the DUT and check
@@ -102,7 +102,7 @@ async def test_real_assign_double(dut):
     assert got == val, "Values didn't match!"
 
 
-@cocotb.test(expect_error=cocotb.SIM_NAME in ["Icarus Verilog"])
+@cocotb.test(expect_error=AttributeError if cocotb.SIM_NAME in ["Icarus Verilog"] else ())
 async def test_real_assign_int(dut):
     """Assign a random integer value to ensure we can write types convertible to
     int, read it back from the DUT and check it matches what we assigned.
diff --git a/tests/test_cases/test_cocotb/test_tests.py b/tests/test_cases/test_cocotb/test_tests.py
--- a/tests/test_cases/test_cocotb/test_tests.py
+++ b/tests/test_cases/test_cocotb/test_tests.py
@@ -14,9 +14,9 @@
 from common import clock_gen
 
 
-@cocotb.test(expect_error=True)
-async def test_syntax_error(dut):
-    """Syntax error in the test"""
+@cocotb.test(expect_error=NameError)
+async def test_error(dut):
+    """Error in the test"""
     await clock_gen(dut.clk)
     fail  # noqa
 
diff --git a/tests/test_cases/test_cocotb/test_timing_triggers.py b/tests/test_cases/test_cocotb/test_timing_triggers.py
--- a/tests/test_cases/test_cocotb/test_timing_triggers.py
+++ b/tests/test_cases/test_cocotb/test_timing_triggers.py
@@ -113,10 +113,8 @@ async def do_test_afterdelay_in_readonly(dut, delay):
     exited = True
 
 
-@cocotb.test(expect_error=True,
-             skip=cocotb.LANGUAGE in ["vhdl"] and cocotb.SIM_NAME.lower().startswith(("riviera")),  # gh-1245
+@cocotb.test(expect_error=TriggerException if cocotb.LANGUAGE in ["verilog"] and cocotb.SIM_NAME.lower().startswith(("riviera")) else (),
              expect_fail=cocotb.SIM_NAME.lower().startswith(("icarus",
-                                                             "riviera",
                                                              "modelsim",
                                                              "ncsim",
                                                              "xmsim")))
@@ -131,12 +129,7 @@ async def test_readwrite_in_readonly(dut):
     assert exited
 
 
-@cocotb.test(expect_error=True,
-             expect_fail=cocotb.SIM_NAME.lower().startswith(("icarus",
-                                                             "riviera",
-                                                             "modelsim",
-                                                             "ncsim",
-                                                             "xmsim")))
+@cocotb.test(expect_error=Exception)
 async def test_cached_write_in_readonly(dut):
     """Test doing invalid sim operation"""
     global exited
diff --git a/tests/test_cases/test_discovery/test_discovery.py b/tests/test_cases/test_discovery/test_discovery.py
--- a/tests/test_cases/test_discovery/test_discovery.py
+++ b/tests/test_cases/test_discovery/test_discovery.py
@@ -68,7 +68,7 @@ async def discover_module_values(dut):
         raise TestFailure("Expected to discover things in the DUT")
 
 
-@cocotb.test(expect_error=True)
+@cocotb.test(expect_error=AttributeError)
 async def discover_value_not_in_dut(dut):
     """Try and get a value from the DUT that is not there"""
     fake_signal = dut.fake_signal
@@ -91,7 +91,7 @@ async def access_signal(dut):
 
 @cocotb.test(
     # Icarus up to (including) 10.3 doesn't support bit-selects, see https://github.com/steveicarus/iverilog/issues/323
-    expect_error=IndexError if (cocotb.SIM_NAME.lower().startswith("icarus") and (IcarusVersion(cocotb.SIM_VERSION) <= IcarusVersion("10.3 (stable)"))) else False,
+    expect_error=IndexError if (cocotb.SIM_NAME.lower().startswith("icarus") and (IcarusVersion(cocotb.SIM_VERSION) <= IcarusVersion("10.3 (stable)"))) else (),
     skip=cocotb.LANGUAGE in ["vhdl"])
 async def access_single_bit(dut):
     """Access a single bit in a vector of the DUT"""
@@ -109,7 +109,7 @@ async def access_single_bit(dut):
 
 @cocotb.test(
     # Icarus up to (including) 10.3 doesn't support bit-selects, see https://github.com/steveicarus/iverilog/issues/323
-    expect_error=IndexError if (cocotb.SIM_NAME.lower().startswith("icarus") and (IcarusVersion(cocotb.SIM_VERSION) <= IcarusVersion("10.3 (stable)"))) else False,
+    expect_error=IndexError if (cocotb.SIM_NAME.lower().startswith("icarus") and (IcarusVersion(cocotb.SIM_VERSION) <= IcarusVersion("10.3 (stable)"))) else (),
     skip=cocotb.LANGUAGE in ["vhdl"])
 async def access_single_bit_assignment(dut):
     """Access a single bit in a vector of the DUT using the assignment mechanism"""
@@ -125,7 +125,7 @@ async def access_single_bit_assignment(dut):
                          dut.stream_out_data_comb.value.integer, (1 << 2)))
 
 
-@cocotb.test(expect_error=True)
+@cocotb.test(expect_error=IndexError)
 async def access_single_bit_erroneous(dut):
     """Access a non-existent single bit"""
     dut._log.info("%s = %d bits" %
@@ -134,7 +134,7 @@ async def access_single_bit_erroneous(dut):
     dut.stream_in_data[bit] <= 1
 
 
-@cocotb.test(expect_error=cocotb.SIM_NAME.lower().startswith(("icarus", "chronologic simulation vcs")),
+@cocotb.test(expect_error=AttributeError if cocotb.SIM_NAME.lower().startswith(("icarus", "chronologic simulation vcs")) else (),
              expect_fail=cocotb.SIM_NAME.lower().startswith(("riviera")) and cocotb.LANGUAGE in ["verilog"])
 async def access_integer(dut):
     """Integer should show as an IntegerObject"""
@@ -250,7 +250,7 @@ async def access_string_vhdl(dut):
 # TODO: add tests for Verilog "string_input_port" and "STRING_LOCALPARAM" (see issue #802)
 
 @cocotb.test(skip=cocotb.LANGUAGE in ["vhdl"] or cocotb.SIM_NAME.lower().startswith(("icarus", "riviera")),
-             expect_error=cocotb.result.TestFailure if cocotb.SIM_NAME.lower().startswith(("xmsim", "ncsim", "modelsim", "chronologic simulation vcs")) else False)
+             expect_error=cocotb.result.TestFailure if cocotb.SIM_NAME.lower().startswith(("xmsim", "ncsim", "modelsim", "chronologic simulation vcs")) else ())
 async def access_const_string_verilog(dut):
     """Access to a const Verilog string."""
     tlog = logging.getLogger("cocotb.test")
@@ -271,7 +271,7 @@ async def access_const_string_verilog(dut):
 
 
 @cocotb.test(skip=cocotb.LANGUAGE in ["vhdl"],
-             expect_error=cocotb.SIM_NAME.lower().startswith("icarus"))
+             expect_error=AttributeError if cocotb.SIM_NAME.lower().startswith("icarus") else ())
 async def access_var_string_verilog(dut):
     """Access to a var Verilog string."""
     tlog = logging.getLogger("cocotb.test")
@@ -369,7 +369,7 @@ async def skip_a_test(dut):
 
 
 @cocotb.test(skip=cocotb.LANGUAGE in ["vhdl"],
-             expect_error=cocotb.SIM_NAME.lower().startswith(("icarus")))
+             expect_error=AttributeError if cocotb.SIM_NAME.lower().startswith(("icarus")) else ())
 async def access_gate(dut):
     """
     Test access to a gate Object
diff --git a/tests/test_cases/test_external/test_external.py b/tests/test_cases/test_external/test_external.py
--- a/tests/test_cases/test_external/test_external.py
+++ b/tests/test_cases/test_external/test_external.py
@@ -90,7 +90,7 @@ async def test_time_in_external(dut):
 
 
 # Cadence simulators: "Unable set up RisingEdge(...) Trigger" with VHDL (see #1076)
-@cocotb.test(expect_error=cocotb.triggers.TriggerException if cocotb.SIM_NAME.startswith(("xmsim", "ncsim")) and cocotb.LANGUAGE in ["vhdl"] else False)
+@cocotb.test(expect_error=cocotb.triggers.TriggerException if cocotb.SIM_NAME.startswith(("xmsim", "ncsim")) and cocotb.LANGUAGE in ["vhdl"] else ())
 async def test_time_in_function(dut):
     """
     Test that an @external function calling back into a cocotb @function
@@ -119,7 +119,7 @@ def wait_cycles_wrapper(dut, n):
 
 
 # Cadence simulators: "Unable set up RisingEdge(...) Trigger" with VHDL (see #1076)
-@cocotb.test(expect_error=cocotb.triggers.TriggerException if cocotb.SIM_NAME.startswith(("xmsim", "ncsim")) and cocotb.LANGUAGE in ["vhdl"] else False)
+@cocotb.test(expect_error=cocotb.triggers.TriggerException if cocotb.SIM_NAME.startswith(("xmsim", "ncsim")) and cocotb.LANGUAGE in ["vhdl"] else ())
 async def test_external_call_return(dut):
     """
     Test ability to await an external function that is not a coroutine using @external
@@ -178,7 +178,7 @@ async def test_function_from_readonly(dut):
 
 
 # Cadence simulators: "Unable set up RisingEdge(...) Trigger" with VHDL (see #1076)
-@cocotb.test(expect_error=cocotb.triggers.TriggerException if cocotb.SIM_NAME.startswith(("xmsim", "ncsim")) and cocotb.LANGUAGE in ["vhdl"] else False)
+@cocotb.test(expect_error=cocotb.triggers.TriggerException if cocotb.SIM_NAME.startswith(("xmsim", "ncsim")) and cocotb.LANGUAGE in ["vhdl"] else ())
 async def test_function_that_awaits(dut):
     """
     Test that @external functions can call @function coroutines that
@@ -192,7 +192,7 @@ async def test_function_that_awaits(dut):
 
 
 # Cadence simulators: "Unable set up RisingEdge(...) Trigger" with VHDL (see #1076)
-@cocotb.test(expect_error=cocotb.triggers.TriggerException if cocotb.SIM_NAME.startswith(("xmsim", "ncsim")) and cocotb.LANGUAGE in ["vhdl"] else False)
+@cocotb.test(expect_error=cocotb.triggers.TriggerException if cocotb.SIM_NAME.startswith(("xmsim", "ncsim")) and cocotb.LANGUAGE in ["vhdl"] else ())
 async def test_await_after_function(dut):
     """
     Test that awaiting a Trigger works after returning
@@ -209,7 +209,7 @@ async def test_await_after_function(dut):
 
 
 # Cadence simulators: "Unable set up RisingEdge(...) Trigger" with VHDL (see #1076)
-@cocotb.test(expect_error=cocotb.triggers.TriggerException if cocotb.SIM_NAME.startswith(("xmsim", "ncsim")) and cocotb.LANGUAGE in ["vhdl"] else False)
+@cocotb.test(expect_error=cocotb.triggers.TriggerException if cocotb.SIM_NAME.startswith(("xmsim", "ncsim")) and cocotb.LANGUAGE in ["vhdl"] else ())
 async def test_external_from_fork(dut):
     """
     Test that @external functions work when awaited from a forked
diff --git a/tests/test_cases/test_failure/test_failure.py b/tests/test_cases/test_failure/test_failure.py
--- a/tests/test_cases/test_failure/test_failure.py
+++ b/tests/test_cases/test_failure/test_failure.py
@@ -11,5 +11,5 @@
 
 
 @cocotb.test()
-async def test_fail():
+async def test_fail(_):
     assert False