Repository: HypothesisWorks/hypothesis

Issue:
Mutable arguments and lazy definition of sampled_from interact poorly
I discovered an unpleasant surprise when debugging #2505 

Consider the following two tests:

```python
from hypothesis import given, strategies as st

@given(st.data())
def test_mutability_1(data):
    x = [1]
    s = st.sampled_from(x)
    x.append(2)
    assert data.draw(s) != 2


@given(st.data())
def test_mutability_2(data):
    x = [1]
    s = st.sampled_from(x)
    assert data.draw(s) != 2
    x.append(2)
    assert data.draw(s) != 2
```

Currently the first one fails and the second one passes.

Why? Well because in the first one the lazy strategy containing the values of `x` is instantiated at the point of the draw after the `append(2)`. in the second one it's instantiated before it.

I think sampled from is the only place where this is currently a significant issue, so this can probably be fixed by removing the `defined_strategy` from sampled_from, but we need to handle some of its special casing seperately to make sure it has a good repr then.


Discussion:

Resolving patch:
diff --git a/hypothesis-python/src/hypothesis/strategies/_internal/core.py b/hypothesis-python/src/hypothesis/strategies/_internal/core.py
--- a/hypothesis-python/src/hypothesis/strategies/_internal/core.py
+++ b/hypothesis-python/src/hypothesis/strategies/_internal/core.py
@@ -191,12 +191,18 @@ def cached_strategy(*args, **kwargs):
     return cached_strategy
 
 
-def base_defines_strategy(force_reusable: bool) -> Callable[[T], T]:
+def base_defines_strategy(
+    force_reusable: bool, *, try_non_lazy: bool = False
+) -> Callable[[T], T]:
     """Returns a decorator for strategy functions.
 
     If force_reusable is True, the generated values are assumed to be
     reusable, i.e. immutable and safe to cache, across multiple test
     invocations.
+
+    If try_non_lazy is True, attempt to execute the strategy definition
+    function immediately, so that a LazyStrategy is only returned if this
+    raises an exception.
     """
 
     def decorator(strategy_definition):
@@ -206,6 +212,16 @@ def decorator(strategy_definition):
 
         @proxies(strategy_definition)
         def accept(*args, **kwargs):
+            if try_non_lazy:
+                # Why not try this unconditionally?  Because we'd end up with very
+                # deep nesting of recursive strategies - better to be lazy unless we
+                # *know* that eager evaluation is the right choice.
+                try:
+                    return strategy_definition(*args, **kwargs)
+                except Exception:
+                    # If invoking the strategy definition raises an exception,
+                    # wrap that up in a LazyStrategy so it happens again later.
+                    pass
             result = LazyStrategy(strategy_definition, args, kwargs)
             if force_reusable:
                 result.force_has_reusable_values = True
@@ -220,6 +236,7 @@ def accept(*args, **kwargs):
 
 defines_strategy = base_defines_strategy(False)
 defines_strategy_with_reusable_values = base_defines_strategy(True)
+defines_strategy_without_laziness = base_defines_strategy(False, try_non_lazy=True)
 
 
 class Nothing(SearchStrategy):
@@ -644,7 +661,7 @@ def sampled_from(elements: Type[enum.Enum]) -> SearchStrategy[Any]:
     pass  # pragma: no cover
 
 
-@defines_strategy  # noqa: F811
+@defines_strategy_without_laziness  # noqa: F811
 def sampled_from(elements):
     """Returns a strategy which generates any value present in ``elements``.
 
@@ -665,13 +682,19 @@ def sampled_from(elements):
         raise InvalidArgument("Cannot sample from a length-zero sequence.")
     if len(values) == 1:
         return just(values[0])
+    if isinstance(elements, type) and issubclass(elements, enum.Enum):
+        repr_ = "sampled_from(%s.%s)" % (elements.__module__, elements.__name__)
+    else:
+        repr_ = "sampled_from(%r)" % (elements,)
     if hasattr(enum, "Flag") and isclass(elements) and issubclass(elements, enum.Flag):
         # Combinations of enum.Flag members are also members.  We generate
         # these dynamically, because static allocation takes O(2^n) memory.
-        return sets(sampled_from(values), min_size=1).map(
+        # LazyStrategy is used for the ease of force_repr.
+        inner = sets(sampled_from(list(values)), min_size=1).map(
             lambda s: reduce(operator.or_, s)
         )
-    return SampledFromStrategy(elements)
+        return LazyStrategy(lambda: inner, args=[], kwargs={}, force_repr=repr_)
+    return SampledFromStrategy(values, repr_)
 
 
 @cacheable
diff --git a/hypothesis-python/src/hypothesis/strategies/_internal/strategies.py b/hypothesis-python/src/hypothesis/strategies/_internal/strategies.py
--- a/hypothesis-python/src/hypothesis/strategies/_internal/strategies.py
+++ b/hypothesis-python/src/hypothesis/strategies/_internal/strategies.py
@@ -13,7 +13,6 @@
 #
 # END HEADER
 
-import enum
 import sys
 import warnings
 from collections import defaultdict
@@ -434,21 +433,14 @@ class SampledFromStrategy(SearchStrategy):
     non-empty subset of the elements.
     """
 
-    def __init__(self, elements):
+    def __init__(self, elements, repr_=None):
         SearchStrategy.__init__(self)
-        self.raw_elements = elements
         self.elements = cu.check_sample(elements, "sampled_from")
         assert self.elements
+        self.repr_ = repr_
 
     def __repr__(self):
-        if isinstance(self.raw_elements, type) and issubclass(
-            self.raw_elements, enum.Enum
-        ):
-            return "sampled_from(%s.%s)" % (
-                self.raw_elements.__module__,
-                self.raw_elements.__name__,
-            )
-        return "sampled_from([%s])" % ", ".join(map(repr, self.elements))
+        return self.repr_ or "sampled_from(%r)" % (list(self.elements),)
 
     def calc_has_reusable_values(self, recur):
         return True


Test patch:
diff --git a/hypothesis-python/tests/cover/test_custom_reprs.py b/hypothesis-python/tests/cover/test_custom_reprs.py
--- a/hypothesis-python/tests/cover/test_custom_reprs.py
+++ b/hypothesis-python/tests/cover/test_custom_reprs.py
@@ -23,6 +23,11 @@ def test_includes_non_default_args_in_repr():
     assert repr(st.integers(min_value=1)) == "integers(min_value=1)"
 
 
+def test_sampled_repr_leaves_range_as_range():
+    huge = 10 ** 100
+    assert repr(st.sampled_from(range(huge))) == "sampled_from(range(0, %s))" % (huge,)
+
+
 def hi(there, stuff):
     return there
 
@@ -35,7 +40,8 @@ def test_supports_positional_and_keyword_args_in_builds():
 
 
 def test_preserves_sequence_type_of_argument():
-    assert repr(st.sampled_from([0])) == "sampled_from([0])"
+    assert repr(st.sampled_from([0, 1])) == "sampled_from([0, 1])"
+    assert repr(st.sampled_from((0, 1))) == "sampled_from((0, 1))"
 
 
 class IHaveABadRepr:
diff --git a/hypothesis-python/tests/cover/test_sampled_from.py b/hypothesis-python/tests/cover/test_sampled_from.py
--- a/hypothesis-python/tests/cover/test_sampled_from.py
+++ b/hypothesis-python/tests/cover/test_sampled_from.py
@@ -86,3 +86,21 @@ def test_max_size_is_respected_with_unique_sampled_from(ls):
 @given(st.lists(st.sampled_from([0, 0.0]), unique=True, min_size=1))
 def test_issue_2247_regression(ls):
     assert len(ls) == 1
+
+
+@given(st.data())
+def test_mutability_1(data):
+    # See https://github.com/HypothesisWorks/hypothesis/issues/2507
+    x = [1]
+    s = st.sampled_from(x)
+    x.append(2)
+    assert data.draw(s) != 2
+
+
+@given(st.data())
+def test_mutability_2(data):
+    x = [1]
+    s = st.sampled_from(x)
+    assert data.draw(s) != 2
+    x.append(2)
+    assert data.draw(s) != 2
diff --git a/hypothesis-python/tests/cover/test_type_lookup.py b/hypothesis-python/tests/cover/test_type_lookup.py
--- a/hypothesis-python/tests/cover/test_type_lookup.py
+++ b/hypothesis-python/tests/cover/test_type_lookup.py
@@ -26,6 +26,7 @@
 from hypothesis.strategies._internal import types
 from hypothesis.strategies._internal.core import _strategies
 from hypothesis.strategies._internal.types import _global_type_lookup
+from tests.common.utils import fails_with
 
 # Build a set of all types output by core strategies
 blacklist = [
@@ -188,9 +189,10 @@ class EmptyEnum(enum.Enum):
     pass
 
 
-def test_error_if_enum_is_empty():
-    with pytest.raises(InvalidArgument):
-        assert st.from_type(EmptyEnum).is_empty
+@fails_with(InvalidArgument)
+@given(st.from_type(EmptyEnum))
+def test_error_if_enum_is_empty(x):
+    pass
 
 
 class BrokenClass:
diff --git a/hypothesis-python/tests/nocover/test_sampled_from.py b/hypothesis-python/tests/nocover/test_sampled_from.py
--- a/hypothesis-python/tests/nocover/test_sampled_from.py
+++ b/hypothesis-python/tests/nocover/test_sampled_from.py
@@ -77,11 +77,17 @@ class AnEnum(enum.Enum):
 
 
 def test_enum_repr_uses_class_not_a_list():
-    strat = st.sampled_from(AnEnum)
-    # Our lazy repr looks exactly like our call
-    lazy_repr = repr(strat)
-    assert lazy_repr == "sampled_from(AnEnum)"
-    # The inner repr should have enough detail to find the class again
+    # The repr should have enough detail to find the class again
     # (which is very useful for the ghostwriter logic we're working on)
-    inner_repr = repr(strat.wrapped_strategy)
-    assert inner_repr == "sampled_from(tests.nocover.test_sampled_from.AnEnum)"
+    lazy_repr = repr(st.sampled_from(AnEnum))
+    assert lazy_repr == "sampled_from(tests.nocover.test_sampled_from.AnEnum)"
+
+
+class AFlag(enum.Flag):
+    a = enum.auto()
+    b = enum.auto()
+
+
+def test_flag_enum_repr_uses_class_not_a_list():
+    lazy_repr = repr(st.sampled_from(AFlag))
+    assert lazy_repr == "sampled_from(tests.nocover.test_sampled_from.AFlag)"