Repository: HypothesisWorks/hypothesis

Issue:
RFC: Disable `HealthCheck.hung_test`?
Now that deadlines are on by default, should we disable the health check that errors when your tests have been running for five minutes?

The reasoning for adding it in was from [discussion when deprecating timeout](https://github.com/HypothesisWorks/hypothesis/pull/580). 

> I’m worried about the case where you’re drawing examples really slowly (or not at all), in which case it seems like the test would just run forever and never stop? Which seems sub-optimal.

Now that we have deadlines and they're enabled by default it means that the only ways to hit the hung test limit are:

1. By setting the deadline to something large or disabling it entirely.
2. By setting `max_examples` to be something high
3. By having a hard to shrink failing test with slow executions (e.g. when shrinking a deadline).

In the first two cases you've basically opted in to slow tests, in the third raising a health check seems unhelpful given that we have an error we could show the user. Perhaps we should introduce a timeout to shrinking with some warning that this is not a fully shrunk example and you should rerun it if you want one?


Discussion:

Resolving patch:
diff --git a/hypothesis-python/src/hypothesis/_settings.py b/hypothesis-python/src/hypothesis/_settings.py
--- a/hypothesis-python/src/hypothesis/_settings.py
+++ b/hypothesis-python/src/hypothesis/_settings.py
@@ -509,7 +509,8 @@ def __repr__(self):
     @classmethod
     def all(cls):
         # type: () -> List[HealthCheck]
-        return list(HealthCheck)
+        deprecated = [HealthCheck.hung_test]
+        return [x for x in list(HealthCheck) if x not in deprecated]
 
     data_too_large = 1
     """Check for when the typical size of the examples you are generating
@@ -529,13 +530,16 @@ def all(cls):
     is unlikely to do what you want)."""
 
     hung_test = 6
-    """Checks if your tests have been running for a very long time."""
+    """This health check is deprecated and no longer has any effect.
+    You can use the ``max_examples`` and ``deadline`` settings together to cap
+    the total runtime of your tests, rather than the previous fixed limit."""
 
     large_base_example = 7
     """Checks if the natural example to shrink towards is very large."""
 
     not_a_test_method = 8
-    """Checks if @given has been applied to a method of unittest.TestCase."""
+    """Checks if :func:`@given <hypothesis.given>` has been applied to a
+    method of :class:`python:unittest.TestCase`."""
 
 
 @unique
@@ -601,6 +605,12 @@ def validate_health_check_suppressions(suppressions):
                 "Non-HealthCheck value %r of type %s is invalid in suppress_health_check."
                 % (s, type(s).__name__)
             )
+        if s is HealthCheck.hung_test:
+            note_deprecation(
+                "HealthCheck.hung_test is deprecated and has no "
+                "effect, as we no longer run this health check.",
+                since="RELEASEDAY",
+            )
     return suppressions
 
 
diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/engine.py b/hypothesis-python/src/hypothesis/internal/conjecture/engine.py
--- a/hypothesis-python/src/hypothesis/internal/conjecture/engine.py
+++ b/hypothesis-python/src/hypothesis/internal/conjecture/engine.py
@@ -27,7 +27,6 @@
 from hypothesis._settings import local_settings
 from hypothesis.internal.compat import (
     Counter,
-    benchmark_time,
     ceil,
     hbytes,
     hrange,
@@ -52,7 +51,6 @@
 __tracebackhide__ = True
 
 
-HUNG_TEST_TIME_LIMIT = 5 * 60
 MAX_SHRINKS = 500
 
 CACHE_RESET_FREQUENCY = 1000
@@ -87,7 +85,6 @@ def __init__(self, test_function, settings=None, random=None, database_key=None)
         self.call_count = 0
         self.event_call_counts = Counter()
         self.valid_examples = 0
-        self.start_time = benchmark_time()
         self.random = random or Random(getrandbits(128))
         self.database_key = database_key
         self.status_runtimes = {}
@@ -116,17 +113,6 @@ def __tree_is_exhausted(self):
         return self.tree.is_exhausted
 
     def test_function(self, data):
-        if benchmark_time() - self.start_time >= HUNG_TEST_TIME_LIMIT:
-            fail_health_check(
-                self.settings,
-                (
-                    "Your test has been running for at least five minutes. This "
-                    "is probably not what you intended, so by default Hypothesis "
-                    "turns it into an error."
-                ),
-                HealthCheck.hung_test,
-            )
-
         self.call_count += 1
         try:
             self._test_function(data)
@@ -725,12 +711,9 @@ def draw_bytes(data, n):
             mutations += 1
 
     def _run(self):
-        self.start_time = benchmark_time()
-
         self.reuse_existing_examples()
         self.generate_new_examples()
         self.shrink_interesting_examples()
-
         self.exit_with(ExitReason.finished)
 
     def shrink_interesting_examples(self):


Test patch:
diff --git a/hypothesis-python/tests/cover/test_conjecture_engine.py b/hypothesis-python/tests/cover/test_conjecture_engine.py
--- a/hypothesis-python/tests/cover/test_conjecture_engine.py
+++ b/hypothesis-python/tests/cover/test_conjecture_engine.py
@@ -19,7 +19,6 @@
 
 import itertools
 import re
-import time
 from random import Random, seed as seed_random
 
 import attr
@@ -451,13 +450,6 @@ def _(data):
         data.draw(SLOW)
 
 
-def test_fails_healthcheck_for_hung_test():
-    @fails_health_check(HealthCheck.hung_test)
-    def _(data):
-        data.draw_bytes(1)
-        time.sleep(3600)
-
-
 @pytest.mark.parametrize("n_large", [1, 5, 8, 15])
 def test_can_shrink_variable_draws(n_large):
     target = 128 * n_large
diff --git a/hypothesis-python/tests/cover/test_deadline.py b/hypothesis-python/tests/cover/test_deadline.py
--- a/hypothesis-python/tests/cover/test_deadline.py
+++ b/hypothesis-python/tests/cover/test_deadline.py
@@ -22,7 +22,7 @@
 import pytest
 
 import hypothesis.strategies as st
-from hypothesis import HealthCheck, given, settings
+from hypothesis import given, settings
 from hypothesis.errors import DeadlineExceeded, Flaky, InvalidArgument
 from tests.common.utils import capture_out, fails_with
 
@@ -85,7 +85,7 @@ def test_keeps_you_well_above_the_deadline():
     seen = set()
     failed_once = [False]
 
-    @settings(deadline=100, suppress_health_check=[HealthCheck.hung_test])
+    @settings(deadline=100)
     @given(st.integers(0, 2000))
     def slow(i):
         # Make sure our initial failure isn't something that immediately goes
diff --git a/hypothesis-python/tests/cover/test_health_checks.py b/hypothesis-python/tests/cover/test_health_checks.py
--- a/hypothesis-python/tests/cover/test_health_checks.py
+++ b/hypothesis-python/tests/cover/test_health_checks.py
@@ -28,7 +28,7 @@
 from hypothesis.errors import FailedHealthCheck, InvalidArgument
 from hypothesis.internal.compat import int_from_bytes
 from hypothesis.searchstrategy.strategies import SearchStrategy
-from tests.common.utils import no_shrink
+from tests.common.utils import checks_deprecated_behaviour, no_shrink
 
 
 def test_slow_generation_fails_a_health_check():
@@ -136,19 +136,9 @@ def a(x):
         a()
 
 
-def test_a_very_slow_test_will_fail_a_health_check():
-    @given(st.integers())
-    @settings(deadline=None)
-    def a(x):
-        time.sleep(1000)
-
-    with raises(FailedHealthCheck):
-        a()
-
-
 def test_the_slow_test_health_check_can_be_disabled():
     @given(st.integers())
-    @settings(suppress_health_check=[HealthCheck.hung_test], deadline=None)
+    @settings(deadline=None)
     def a(x):
         time.sleep(1000)
 
@@ -200,6 +190,11 @@ def test_it_is_an_error_to_suppress_non_iterables():
         settings(suppress_health_check=1)
 
 
+@checks_deprecated_behaviour
+def test_hung_test_is_deprecated():
+    settings(suppress_health_check=[HealthCheck.hung_test])
+
+
 def test_it_is_an_error_to_suppress_non_healthchecks():
     with raises(InvalidArgument):
         settings(suppress_health_check=[1])