Repository: HypothesisWorks/hypothesis (Python)

Issue:
no example run should mark test as skipped
Currently ( `hypothesis        6.45.1`) when no example is run, the test is marked as success.
One obvious example:
```
from hypothesis import given, settings, example
import hypothesis.strategies as st



@given(my_int=st.integers())
@settings(phases=[])
@example(my_int=51)
def test_integers(my_int):
    assert my_int == 42
```
```
================================================= test session starts ==================================================
platform linux -- Python 3.9.7, pytest-7.1.2, pluggy-1.0.0
rootdir: /home/alexv/tmp/hypofix
plugins: hypothesis-6.45.1
collected 1 item

no_test.py .                                                                                                     [100%]

================================================== 1 passed in 0.11s ===================================================
```

Instead the test should be marked as skipped, as it was never run.

I am not sure how feasible it is, as there are various combinations of `failures_to_reproduce x explicit_examples x phases_configured) that can bring about this behavior.

However when this kind of false positive happens, it is easy to get caught by it, when one is not deeply aware of the details of the test and its configuration...



Discussion:
We can have Hypothesis increment an integer `_n_valid_or_failing_examples` (for example), and then check in the pytest plugin to skip if this is zero.



Patch:
diff --git a/hypothesis-python/RELEASE.rst b/hypothesis-python/RELEASE.rst
new file mode 100644
index 0000000000..c61863c9d3
--- /dev/null
+++ b/hypothesis-python/RELEASE.rst
@@ -0,0 +1,7 @@
+RELEASE_TYPE: minor
+
+This release raises :class:`~unittest.SkipTest` for which never executed any
+examples, for example because the :obj:`~hypothesis.settings.phases` setting
+excluded the :obj:`~hypothesis.Phase.explicit`, :obj:`~hypothesis.Phase.reuse`,
+and :obj:`~hypothesis.Phase.generate` phases.  This helps to avoid cases where
+broken tests appear to pass, because they didn't actually execute (:issue:`3328`).
diff --git a/hypothesis-python/src/hypothesis/core.py b/hypothesis-python/src/hypothesis/core.py
index 434c0e553e..627511b2d9 100644
--- a/hypothesis-python/src/hypothesis/core.py
+++ b/hypothesis-python/src/hypothesis/core.py
@@ -18,6 +18,7 @@
 import sys
 import time
 import types
+import unittest
 import warnings
 import zlib
 from collections import defaultdict
@@ -556,6 +557,7 @@ def __init__(
         self.__was_flaky = False
         self.random = random
         self.__test_runtime = None
+        self.ever_executed = False
 
         self.is_find = getattr(wrapped_test, "_hypothesis_internal_is_find", False)
         self.wrapped_test = wrapped_test
@@ -586,6 +588,7 @@ def execute_once(
         swallowed the corresponding control exception.
         """
 
+        self.ever_executed = True
         data.is_find = self.is_find
 
         text_repr = None
@@ -1189,9 +1192,17 @@ def wrapped_test(*arguments, **kwargs):
             # The next step is to use the Conjecture engine to run the test on
             # many different inputs.
 
+            ran_explicit_examples = Phase.explicit in state.settings.phases and getattr(
+                wrapped_test, "hypothesis_explicit_examples", ()
+            )
+            SKIP_BECAUSE_NO_EXAMPLES = unittest.SkipTest(
+                "Hypothesis has been told to run no examples for this test."
+            )
             if not (
                 Phase.reuse in settings.phases or Phase.generate in settings.phases
             ):
+                if not ran_explicit_examples:
+                    raise SKIP_BECAUSE_NO_EXAMPLES
                 return
 
             try:
@@ -1236,6 +1247,9 @@ def wrapped_test(*arguments, **kwargs):
                     )
                     raise the_error_hypothesis_found
 
+            if not (ran_explicit_examples or state.ever_executed):
+                raise SKIP_BECAUSE_NO_EXAMPLES
+
         def _get_fuzz_target() -> Callable[
             [Union[bytes, bytearray, memoryview, BinaryIO]], Optional[bytes]
         ]:


Test patch:
diff --git a/hypothesis-python/tests/cover/test_core.py b/hypothesis-python/tests/cover/test_core.py
index a470d4d848..c9b25987dd 100644
--- a/hypothesis-python/tests/cover/test_core.py
+++ b/hypothesis-python/tests/cover/test_core.py
@@ -8,10 +8,13 @@
 # v. 2.0. If a copy of the MPL was not distributed with this file, You can
 # obtain one at https://mozilla.org/MPL/2.0/.
 
+import unittest
+
 import pytest
 from _pytest.outcomes import Failed, Skipped
 
-from hypothesis import find, given, reject, settings, strategies as s
+from hypothesis import Phase, example, find, given, reject, settings, strategies as s
+from hypothesis.database import InMemoryExampleDatabase
 from hypothesis.errors import InvalidArgument, NoSuchExample, Unsatisfiable
 
 
@@ -111,3 +114,32 @@ def test_method_with_bad_strategy(self, x):
     instance = TestStrategyValidation()
     with pytest.raises(InvalidArgument):
         instance.test_method_with_bad_strategy()
+
+
+@example(1)
+@given(s.integers())
+@settings(phases=[Phase.target, Phase.shrink, Phase.explain])
+def no_phases(_):
+    raise Exception
+
+
+@given(s.integers())
+@settings(phases=[Phase.explicit])
+def no_explicit(_):
+    raise Exception
+
+
+@given(s.integers())
+@settings(phases=[Phase.reuse], database=InMemoryExampleDatabase())
+def empty_db(_):
+    raise Exception
+
+
+@pytest.mark.parametrize(
+    "test_fn",
+    [no_phases, no_explicit, empty_db],
+    ids=lambda t: t.__name__,
+)
+def test_non_executed_tests_raise_skipped(test_fn):
+    with pytest.raises(unittest.SkipTest):
+        test_fn()
diff --git a/hypothesis-python/tests/cover/test_fuzz_one_input.py b/hypothesis-python/tests/cover/test_fuzz_one_input.py
index dd175ec599..9918cc3223 100644
--- a/hypothesis-python/tests/cover/test_fuzz_one_input.py
+++ b/hypothesis-python/tests/cover/test_fuzz_one_input.py
@@ -9,6 +9,7 @@
 # obtain one at https://mozilla.org/MPL/2.0/.
 
 import io
+import unittest
 from operator import attrgetter
 
 import pytest
@@ -45,7 +46,8 @@ def test(s):
 
     # Before running fuzz_one_input, there's nothing in `db`, and so the test passes
     # (because example generation is disabled by the custom settings)
-    test()
+    with pytest.raises(unittest.SkipTest):  # because this generates no examples
+        test()
     assert len(seen) == 0
 
     # If we run a lot of random bytestrings through fuzz_one_input, we'll eventually