Repository: HypothesisWorks/hypothesis

Issue:
`test_can_generate_unique_columns` is flaky
In a [`py11-check-niche` job](https://github.com/HypothesisWorks/hypothesis/actions/runs/5053311410/jobs/9067036370?pr=3645) we have `tests/pandas/test_data_frame.py::test_can_generate_unique_columns` failing with `Data generation is extremely slow: Only produced 9 valid examples in 1.01 seconds (92 invalid ones and 0 exceeded maximum size).`

For reference

https://github.com/HypothesisWorks/hypothesis/blob/661af850bbfcb091a820eb16c84f56048c8e21c8/hypothesis-python/tests/pandas/test_data_frame.py#L245-L253

The `filter_too_much` suppression makes sense with the hard-to-satisfy `pdst.column(elements=st.integers(0, 9), fill=None, unique=True)`, but I wonder if sometimes its just going to generate such a number of rejected example that the test warrants a `too_slow` suppression too.


Discussion:
Yeah, let's just suppress.  See also https://github.com/HypothesisWorks/hypothesis/issues/3647#issuecomment-1564219900.

Resolving patch:
diff --git a/tooling/src/hypothesistooling/__main__.py b/tooling/src/hypothesistooling/__main__.py
--- a/tooling/src/hypothesistooling/__main__.py
+++ b/tooling/src/hypothesistooling/__main__.py
@@ -13,7 +13,7 @@
 import re
 import subprocess
 import sys
-from glob import glob
+from pathlib import Path
 
 import requests
 from coverage.config import CoverageConfig
@@ -241,23 +241,29 @@ def compile_requirements(upgrade=False):
     else:
         extra = []
 
-    for f in glob(os.path.join("requirements", "*.in")):
-        base, _ = os.path.splitext(f)
+    for f in Path("requirements").glob("*.in"):
+        out_file = f.with_suffix(".txt")
         pip_tool(
             "pip-compile",
             "--allow-unsafe",  # future default, not actually unsafe
             "--resolver=backtracking",  # new pip resolver, default in pip-compile 7+
             *extra,
-            f,
+            str(f),
             "hypothesis-python/setup.py",
             "--output-file",
-            base + ".txt",
+            str(out_file),
             cwd=tools.ROOT,
             env={
                 "CUSTOM_COMPILE_COMMAND": "./build.sh upgrade-requirements",
                 **os.environ,
             },
         )
+        # Check that we haven't added anything to output files without adding to inputs
+        out_pkgs = out_file.read_text()
+        for p in f.read_text().splitlines():
+            p = p.lower().replace("_", "-")
+            if re.fullmatch(r"[a-z-]+", p):
+                assert p + "==" in out_pkgs, f"Package `{p}` deleted from {out_file}"
 
 
 def update_python_versions():


Test patch:
diff --git a/hypothesis-python/tests/cover/test_complex_numbers.py b/hypothesis-python/tests/cover/test_complex_numbers.py
--- a/hypothesis-python/tests/cover/test_complex_numbers.py
+++ b/hypothesis-python/tests/cover/test_complex_numbers.py
@@ -13,7 +13,7 @@
 
 import pytest
 
-from hypothesis import given, reject, strategies as st
+from hypothesis import HealthCheck, given, reject, settings, strategies as st
 from hypothesis.errors import InvalidArgument
 from hypothesis.strategies import complex_numbers
 
@@ -62,6 +62,7 @@ def test_max_magnitude_zero(val):
     assert val == 0
 
 
+@settings(suppress_health_check=[HealthCheck.too_slow])
 @given(st.data(), st.integers(-5, 5).map(lambda x: 10**x))
 def test_min_magnitude_respected(data, mag):
     c = data.draw(complex_numbers(min_magnitude=mag))
diff --git a/hypothesis-python/tests/numpy/test_gen_data.py b/hypothesis-python/tests/numpy/test_gen_data.py
--- a/hypothesis-python/tests/numpy/test_gen_data.py
+++ b/hypothesis-python/tests/numpy/test_gen_data.py
@@ -869,6 +869,7 @@ def test_mutually_broadcastable_shapes_shrinking_with_singleton_out_of_bounds(
         assert smallest == (min_side,) * min_dims
 
 
+@settings(suppress_health_check=[HealthCheck.too_slow])
 @given(
     num_shapes=st.integers(1, 4),
     min_dims=st.integers(1, 32),
diff --git a/hypothesis-python/tests/pandas/test_data_frame.py b/hypothesis-python/tests/pandas/test_data_frame.py
--- a/hypothesis-python/tests/pandas/test_data_frame.py
+++ b/hypothesis-python/tests/pandas/test_data_frame.py
@@ -242,7 +242,7 @@ def test_will_fill_missing_columns_in_tuple_row(df):
         assert d == 7
 
 
-@settings(suppress_health_check=[HealthCheck.filter_too_much])
+@settings(suppress_health_check=[HealthCheck.filter_too_much, HealthCheck.too_slow])
 @given(
     pdst.data_frames(
         index=pdst.range_indexes(10, 10),