Repository: HypothesisWorks/hypothesis

Issue:
`from_type(Set[Decimal])` raises an internal error because signalling NaN is not hashable
I've gotten `from_type(Set[Hashable])` to reliably fail during generation. It seems that `Hashable` generates `Decimal` objects, which is fine. However, hypothesis fails when trying to hash signaling NaNs.

I'm not too familiar with the inner workings of strategy generation, but I think it would be reasonable to "turn off" signaling NaNs in composite strategies whenever they might need to be hashed (`sets`, `frozensets`, the keys of `dictionaries`, etc...).

Here's a minimal example.

```py
Python 3.7.4 (default, Sep  7 2019, 18:27:02) 
[Clang 10.0.1 (clang-1001.0.46.4)] on darwin
Type "help", "copyright", "credits" or "license" for more information.
>>> import hypothesis
>>> hypothesis.__version__
'5.1.5'
>>> from hypothesis.strategies import decimals, sets
>>> s = sets(decimals())
>>> s.example()
{Decimal('-NaN'), Decimal('-0.5'), Decimal('0.294355'), Decimal('Infinity'), Decimal('3.00472547')}
>>> s.example()
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/usr/local/lib/python3.7/site-packages/hypothesis/strategies/_internal/strategies.py", line 314, in example
    example_generating_inner_function()
  File "/usr/local/lib/python3.7/site-packages/hypothesis/strategies/_internal/strategies.py", line 302, in example_generating_inner_function
    @settings(
  File "/usr/local/lib/python3.7/site-packages/hypothesis/core.py", line 1080, in wrapped_test
    raise the_error_hypothesis_found
  File "/usr/local/lib/python3.7/site-packages/hypothesis/core.py", line 1049, in wrapped_test
    state.run_engine()
  File "/usr/local/lib/python3.7/site-packages/hypothesis/core.py", line 723, in run_engine
    runner.run()
  File "/usr/local/lib/python3.7/site-packages/hypothesis/internal/conjecture/engine.py", line 420, in run
    self._run()
  File "/usr/local/lib/python3.7/site-packages/hypothesis/internal/conjecture/engine.py", line 815, in _run
    self.generate_new_examples()
  File "/usr/local/lib/python3.7/site-packages/hypothesis/internal/conjecture/engine.py", line 667, in generate_new_examples
    self.test_function(data)
  File "/usr/local/lib/python3.7/site-packages/hypothesis/internal/conjecture/engine.py", line 155, in test_function
    self.__stoppable_test_function(data)
  File "/usr/local/lib/python3.7/site-packages/hypothesis/internal/conjecture/engine.py", line 137, in __stoppable_test_function
    self._test_function(data)
  File "/usr/local/lib/python3.7/site-packages/hypothesis/core.py", line 676, in _execute_once_for_engine
    escalate_hypothesis_internal_error()
  File "/usr/local/lib/python3.7/site-packages/hypothesis/core.py", line 647, in _execute_once_for_engine
    result = self.execute_once(data)
  File "/usr/local/lib/python3.7/site-packages/hypothesis/core.py", line 602, in execute_once
    result = self.test_runner(data, run)
  File "/usr/local/lib/python3.7/site-packages/hypothesis/executors.py", line 52, in default_new_style_executor
    return function(data)
  File "/usr/local/lib/python3.7/site-packages/hypothesis/core.py", line 543, in run
    args, kwargs = data.draw(self.search_strategy)
  File "/usr/local/lib/python3.7/site-packages/hypothesis/internal/conjecture/data.py", line 879, in draw
    return strategy.do_draw(self)
  File "/usr/local/lib/python3.7/site-packages/hypothesis/strategies/_internal/collections.py", line 57, in do_draw
    return tuple(data.draw(e) for e in self.element_strategies)
  File "/usr/local/lib/python3.7/site-packages/hypothesis/strategies/_internal/collections.py", line 57, in <genexpr>
    return tuple(data.draw(e) for e in self.element_strategies)
  File "/usr/local/lib/python3.7/site-packages/hypothesis/internal/conjecture/data.py", line 874, in draw
    return strategy.do_draw(self)
  File "/usr/local/lib/python3.7/site-packages/hypothesis/strategies/_internal/strategies.py", line 639, in do_draw
    result = self.pack(data.draw(self.mapped_strategy))
  File "/usr/local/lib/python3.7/site-packages/hypothesis/internal/conjecture/data.py", line 874, in draw
    return strategy.do_draw(self)
  File "/usr/local/lib/python3.7/site-packages/hypothesis/strategies/_internal/lazy.py", line 150, in do_draw
    return data.draw(self.wrapped_strategy)
  File "/usr/local/lib/python3.7/site-packages/hypothesis/internal/conjecture/data.py", line 874, in draw
    return strategy.do_draw(self)
  File "/usr/local/lib/python3.7/site-packages/hypothesis/strategies/_internal/strategies.py", line 639, in do_draw
    result = self.pack(data.draw(self.mapped_strategy))
  File "/usr/local/lib/python3.7/site-packages/hypothesis/internal/conjecture/data.py", line 874, in draw
    return strategy.do_draw(self)
  File "/usr/local/lib/python3.7/site-packages/hypothesis/strategies/_internal/collections.py", line 57, in do_draw
    return tuple(data.draw(e) for e in self.element_strategies)
  File "/usr/local/lib/python3.7/site-packages/hypothesis/strategies/_internal/collections.py", line 57, in <genexpr>
    return tuple(data.draw(e) for e in self.element_strategies)
  File "/usr/local/lib/python3.7/site-packages/hypothesis/internal/conjecture/data.py", line 874, in draw
    return strategy.do_draw(self)
  File "/usr/local/lib/python3.7/site-packages/hypothesis/strategies/_internal/lazy.py", line 150, in do_draw
    return data.draw(self.wrapped_strategy)
  File "/usr/local/lib/python3.7/site-packages/hypothesis/internal/conjecture/data.py", line 874, in draw
    return strategy.do_draw(self)
  File "/usr/local/lib/python3.7/site-packages/hypothesis/strategies/_internal/strategies.py", line 639, in do_draw
    result = self.pack(data.draw(self.mapped_strategy))
  File "/usr/local/lib/python3.7/site-packages/hypothesis/internal/conjecture/data.py", line 874, in draw
    return strategy.do_draw(self)
  File "/usr/local/lib/python3.7/site-packages/hypothesis/strategies/_internal/lazy.py", line 150, in do_draw
    return data.draw(self.wrapped_strategy)
  File "/usr/local/lib/python3.7/site-packages/hypothesis/internal/conjecture/data.py", line 874, in draw
    return strategy.do_draw(self)
  File "/usr/local/lib/python3.7/site-packages/hypothesis/strategies/_internal/collections.py", line 158, in do_draw
    data=data, filter_strategy=filtered
  File "/usr/local/lib/python3.7/site-packages/hypothesis/strategies/_internal/strategies.py", line 352, in do_filtered_draw
    return filter_strategy.default_do_filtered_draw(data)
  File "/usr/local/lib/python3.7/site-packages/hypothesis/strategies/_internal/strategies.py", line 729, in default_do_filtered_draw
    if self.condition(value):
  File "/usr/local/lib/python3.7/site-packages/hypothesis/strategies/_internal/collections.py", line 153, in <lambda>
    key(val) not in seen for (key, seen) in zip(self.keys, seen_sets)
  File "/usr/local/lib/python3.7/site-packages/hypothesis/strategies/_internal/collections.py", line 153, in <genexpr>
    key(val) not in seen for (key, seen) in zip(self.keys, seen_sets)
TypeError: Cannot hash a signaling NaN value
```

Thanks for such an awesome library! If this is an easy fix, I'm happy to take it... but a quick scan of the relevant code seemed a bit complex.


Discussion:
Of course an easy short-term fix is to un-register `decimals` as `Hashable`...
Hi Brandt - thanks for raising this!

`from_type(Hashable)` generating things which raise an error when hashed is working as intended - `Decimal("snan")` is still *of a hashable type*, and the fact that you get an error is incidental.  If you need hashable *values*, you can filter the strategy.

`from_type(Set[Hashable])` is different though - I think we should deal with this internally, because you've asked for *sets* and raising an error in construction isn't giving you a set.  Doing this in the general case without hiding real problems seems very very difficult, but just checking for hashability should be tractable and I think that will cover most constraints on the elements of collection types.

Resolving patch:
diff --git a/hypothesis-python/src/hypothesis/strategies/_internal/types.py b/hypothesis-python/src/hypothesis/strategies/_internal/types.py
--- a/hypothesis-python/src/hypothesis/strategies/_internal/types.py
+++ b/hypothesis-python/src/hypothesis/strategies/_internal/types.py
@@ -406,21 +406,29 @@ def resolve_List(thing):
     return st.lists(st.from_type(thing.__args__[0]))
 
 
+def _can_hash(val):
+    try:
+        hash(val)
+        return True
+    except Exception:
+        return False
+
+
 @register(typing.Set, st.builds(set))
 def resolve_Set(thing):
-    return st.sets(st.from_type(thing.__args__[0]))
+    return st.sets(st.from_type(thing.__args__[0]).filter(_can_hash))
 
 
 @register(typing.FrozenSet, st.builds(frozenset))
 def resolve_FrozenSet(thing):
-    return st.frozensets(st.from_type(thing.__args__[0]))
+    return st.frozensets(st.from_type(thing.__args__[0]).filter(_can_hash))
 
 
 @register(typing.Dict, st.builds(dict))
 def resolve_Dict(thing):
     # If thing is a Collection instance, we need to fill in the values
     keys_vals = [st.from_type(t) for t in thing.__args__] * 2
-    return st.dictionaries(keys_vals[0], keys_vals[1])
+    return st.dictionaries(keys_vals[0].filter(_can_hash), keys_vals[1])
 
 
 @register("DefaultDict", st.builds(collections.defaultdict))
@@ -435,7 +443,9 @@ def resolve_ItemsView(thing):
 
 @register(typing.KeysView, st.builds(dict).map(dict.keys))
 def resolve_KeysView(thing):
-    return st.dictionaries(st.from_type(thing.__args__[0]), st.none()).map(dict.keys)
+    return st.dictionaries(
+        st.from_type(thing.__args__[0]).filter(_can_hash), st.none()
+    ).map(dict.keys)
 
 
 @register(typing.ValuesView, st.builds(dict).map(dict.values))


Test patch:
diff --git a/hypothesis-python/tests/cover/test_lookup.py b/hypothesis-python/tests/cover/test_lookup.py
--- a/hypothesis-python/tests/cover/test_lookup.py
+++ b/hypothesis-python/tests/cover/test_lookup.py
@@ -632,3 +632,21 @@ def test_can_cast():
 def test_timezone_lookup(type_):
     assert issubclass(type_, datetime.tzinfo)
     assert_all_examples(st.from_type(type_), lambda t: isinstance(t, datetime.timezone))
+
+
+@pytest.mark.parametrize(
+    "typ",
+    [
+        typing.Set[typing.Hashable],
+        typing.FrozenSet[typing.Hashable],
+        typing.Dict[typing.Hashable, int],
+    ],
+)
+def test_generic_collections_only_use_hashable_elements(typ):
+    assert_all_examples(from_type(typ), lambda x: True)
+
+
+def test_hashable_type_unhashable_value():
+    # Decimal("snan") is not hashable; we should be able to generate it.
+    # See https://github.com/HypothesisWorks/hypothesis/issues/2320
+    find_any(from_type(typing.Hashable), lambda x: not types._can_hash(x))