Repository: HypothesisWorks/hypothesis
Pull request: Clean up tests/cover/test_numerics.py
This:

* Replaces use of `@fails` to assert the ability to find certain examples with the more modern `find_any` 
* Replaces all use of math functions (which convert to floats) with the corresponding decimal methods

As a result fixes #1033 

Problem statement:
test_decimals_include_nan is flaky
```
---------------------------------- Hypothesis ----------------------------------
You can add @seed(327206777863050896557169937789875330632) to this test or run pytest with --hypothesis-seed=327206777863050896557169937789875330632 to reproduce this failure.
__________________________ test_decimals_include_nan ___________________________
[gw8] linux -- Python 3.6.3 /usr/bin/python
tests/cover/test_numerics.py:88: in test_decimals_include_nan
    @given(decimals())
tests/common/utils.py:64: in inverted_test
    f(*arguments, **kwargs)
tests/cover/test_numerics.py:88: in test_decimals_include_nan
    @given(decimals())
E   hypothesis.errors.MultipleFailures: Hypothesis found 2 distinct failures.
---------------------------------- Hypothesis ----------------------------------
Falsifying example: test_decimals_include_nan(x=Decimal('sNaN'))
Traceback (most recent call last):
  File "/build/python-hypothesis/src/hypothesis-python-3.44.1/src/hypothesis/core.py", line 818, in run
    falsifying_example.__expected_traceback,
  File "/build/python-hypothesis/src/hypothesis-python-3.44.1/src/hypothesis/core.py", line 579, in execute
    result = self.test_runner(data, run)
  File "/build/python-hypothesis/src/hypothesis-python-3.44.1/src/hypothesis/executors.py", line 58, in default_new_style_executor
    return function(data)
  File "/build/python-hypothesis/src/hypothesis-python-3.44.1/src/hypothesis/core.py", line 571, in run
    return test(*args, **kwargs)
  File "/build/python-hypothesis/src/hypothesis-python-3.44.1/tests/cover/test_numerics.py", line 88, in test_decimals_include_nan
    @given(decimals())
  File "/build/python-hypothesis/src/hypothesis-python-3.44.1/src/hypothesis/core.py", line 518, in test
    result = self.test(*args, **kwargs)
  File "/build/python-hypothesis/src/hypothesis-python-3.44.1/tests/cover/test_numerics.py", line 90, in test_decimals_include_nan
    assert not math.isnan(x)
  File "/usr/lib/python3.6/_pydecimal.py", line 1651, in __float__
    raise ValueError("Cannot convert signaling NaN to float")
ValueError: Cannot convert signaling NaN to float

Falsifying example: test_decimals_include_nan(x=Decimal('NaN'))
Traceback (most recent call last):
  File "/build/python-hypothesis/src/hypothesis-python-3.44.1/src/hypothesis/core.py", line 818, in run
    falsifying_example.__expected_traceback,
  File "/build/python-hypothesis/src/hypothesis-python-3.44.1/src/hypothesis/core.py", line 579, in execute
    result = self.test_runner(data, run)
  File "/build/python-hypothesis/src/hypothesis-python-3.44.1/src/hypothesis/executors.py", line 58, in default_new_style_executor
    return function(data)
  File "/build/python-hypothesis/src/hypothesis-python-3.44.1/src/hypothesis/core.py", line 571, in run
    return test(*args, **kwargs)
  File "/build/python-hypothesis/src/hypothesis-python-3.44.1/tests/cover/test_numerics.py", line 88, in test_decimals_include_nan
    @given(decimals())
  File "/build/python-hypothesis/src/hypothesis-python-3.44.1/src/hypothesis/core.py", line 518, in test
    result = self.test(*args, **kwargs)
  File "/build/python-hypothesis/src/hypothesis-python-3.44.1/tests/cover/test_numerics.py", line 90, in test_decimals_include_nan
    assert not math.isnan(x)
AssertionError: assert not True
 +  where True = <built-in function isnan>(Decimal('NaN'))
 +    where <built-in function isnan> = math.isnan
```

Hints from issue discussion:
I think this is a genuine bug in the test. `math.isnan(x)` is not the correct way to check if a decimal is NaN, that would be `x.is_nan()`.
I'm curious as to why your build is so much better at finding flaky test bugs than ours is though given that it's running the same tests!

Commit message: Remove remaining use of math on decimal

Patch to review:
diff --git a/tests/cover/test_numerics.py b/tests/cover/test_numerics.py
index 05dc8ab508..afd95df6e3 100644
--- a/tests/cover/test_numerics.py
+++ b/tests/cover/test_numerics.py
@@ -17,14 +17,13 @@
 
 from __future__ import division, print_function, absolute_import
 
-import math
 import decimal
 
 import pytest
 
 from hypothesis import given, assume, reject
 from hypothesis.errors import InvalidArgument
-from tests.common.utils import fails
+from tests.common.debug import find_any
 from hypothesis.strategies import data, none, tuples, decimals, integers, \
     fractions
 
@@ -72,11 +71,11 @@ def test_fuzz_decimals_bounds(data):
         assert val.as_tuple().exponent == -places
 
 
-@fails
-@given(decimals())
-def test_all_decimals_can_be_exact_floats(x):
-    assume(x.is_finite())
-    assert decimal.Decimal(float(x)) == x
+def test_all_decimals_can_be_exact_floats():
+    find_any(
+        decimals(),
+        lambda x: assume(x.is_finite()) and decimal.Decimal(float(x)) == x
+    )
 
 
 @given(fractions(), fractions(), fractions())
@@ -84,28 +83,25 @@ def test_fraction_addition_is_well_behaved(x, y, z):
     assert x + y + z == y + x + z
 
 
-@fails
-@given(decimals())
-def test_decimals_include_nan(x):
-    assert not math.isnan(x)
+def test_decimals_include_nan():
+    find_any(decimals(), lambda x: x.is_nan())
 
 
-@fails
-@given(decimals())
-def test_decimals_include_inf(x):
-    assume(not x.is_snan())
-    assert not math.isinf(x)
+def test_decimals_include_inf():
+    find_any(
+        decimals(),
+        lambda x: assume(not x.is_snan()) and x.is_infinite()
+    )
 
 
 @given(decimals(allow_nan=False))
 def test_decimals_can_disallow_nan(x):
-    assert not math.isnan(x)
+    assert not x.is_nan()
 
 
 @given(decimals(allow_infinity=False))
 def test_decimals_can_disallow_inf(x):
-    assume(not x.is_snan())
-    assert not math.isinf(x)
+    assert not x.is_infinite()
 
 
 @pytest.mark.parametrize('places', range(10))


Code review comments:
diff_hunk: @@ -72,40 +71,37 @@ def test_fuzz_decimals_bounds(data):
         assert val.as_tuple().exponent == -places
 
 
-@fails
-@given(decimals())
-def test_all_decimals_can_be_exact_floats(x):
-    assume(x.is_finite())
-    assert decimal.Decimal(float(x)) == x
+def test_all_decimals_can_be_exact_floats():
+    find_any(
+        decimals(),
+        lambda x: assume(x.is_finite()) and decimal.Decimal(float(x)) == x
+    )
 
 
 @given(fractions(), fractions(), fractions())
 def test_fraction_addition_is_well_behaved(x, y, z):
     assert x + y + z == y + x + z
 
 
-@fails
-@given(decimals())
-def test_decimals_include_nan(x):
-    assert not math.isnan(x)
+def test_decimals_include_nan():
+    find_any(decimals(), lambda x: x.is_nan())
 
 
-@fails
-@given(decimals())
-def test_decimals_include_inf(x):
-    assume(not x.is_snan())
-    assert not math.isinf(x)
+def test_decimals_include_inf():
+    find_any(
+        decimals(),
+        lambda x: assume(not x.is_snan()) and x.is_infinite()
path: tests/cover/test_numerics.py
text: @user1:
You probably mean `not x.is_nan()` here - the extra `s` restricts the test to signalling nans, and it could therefore pass without finding an infinity.

@author:
Actually it looks like I mean neither - decimal NaNs don't return True for is_infinite! Have updated the test.

@user1:
...true, they don't.  Merge when the tests pass! 😄