Repository: HypothesisWorks/hypothesis

Issue:
Deprecate non-sequence inputs to the permutations() strategy
`permutations({1, 2, 3})` will generate permutations of the input set, *but* the minimal value that non-sequence inputs will vary between runs - because it is cast to `list` without a stable ordering.  This problem has previously been addressed for `sampled_from()` - see issue #423 for more on the motivation, and #486 for the initial implementation of the fix.

Suggested approach: extend `hypothesis.internal.conjecture.utils.check_sample` to support a permutations-specific deprecation message, and then plug it in similarly to `sampled_from`.

If you would like to try fixing this issue, feel free to ask for clarification on or assistance with anything at all and I'll do my best to help you out 😄 


Discussion:

Resolving patch:
diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/utils.py b/hypothesis-python/src/hypothesis/internal/conjecture/utils.py
--- a/hypothesis-python/src/hypothesis/internal/conjecture/utils.py
+++ b/hypothesis-python/src/hypothesis/internal/conjecture/utils.py
@@ -114,7 +114,7 @@ def centered_integer_range(data, lower, upper, center):
     ndarray = ()
 
 
-def check_sample(values):
+def check_sample(values, strategy_name):
     if isinstance(values, ndarray):
         if values.ndim != 1:
             note_deprecation((
@@ -128,16 +128,16 @@ def check_sample(values):
             ).format(ndim=values.ndim, shape=values.shape))
     elif not isinstance(values, (OrderedDict, Sequence, enum.EnumMeta)):
         note_deprecation(
-            ('Cannot sample from %r, not a sequence.  ' % (values,)) +
-            'Hypothesis goes to some length to ensure that sampling an '
-            'element from a collection (with `sampled_from` or `choices`) is '
-            'replayable and can be minimised.  To replay a saved example, '
-            'the sampled values must have the same iteration order on every '
-            'run - ruling out sets, dicts, etc due to hash randomisation.  '
-            'Most cases can simply use `sorted(values)`, but mixed types or '
-            'special values such as math.nan require careful handling - and '
-            'note that when simplifying an example, Hypothesis treats '
-            'earlier values as simpler.')
+            'Cannot sample from {values}, not an ordered collection. '
+            'Hypothesis goes to some length to ensure that the {strategy} '
+            'strategy has stable results between runs. To replay a saved '
+            'example, the sampled values must have the same iteration order '
+            'on every run - ruling out sets, dicts, etc due to hash '
+            'randomisation. Most cases can simply use `sorted(values)`, but '
+            'mixed types or special values such as math.nan require careful '
+            'handling - and note that when simplifying an example, '
+            'Hypothesis treats earlier values as simpler.'.format(
+                values=repr(values), strategy=strategy_name))
     return tuple(values)
 
 
diff --git a/hypothesis-python/src/hypothesis/searchstrategy/misc.py b/hypothesis-python/src/hypothesis/searchstrategy/misc.py
--- a/hypothesis-python/src/hypothesis/searchstrategy/misc.py
+++ b/hypothesis-python/src/hypothesis/searchstrategy/misc.py
@@ -87,7 +87,7 @@ class SampledFromStrategy(SearchStrategy):
 
     def __init__(self, elements):
         SearchStrategy.__init__(self)
-        self.elements = d.check_sample(elements)
+        self.elements = d.check_sample(elements, 'sampled_from')
         assert self.elements
 
     def calc_has_reusable_values(self, recur):
diff --git a/hypothesis-python/src/hypothesis/strategies.py b/hypothesis-python/src/hypothesis/strategies.py
--- a/hypothesis-python/src/hypothesis/strategies.py
+++ b/hypothesis-python/src/hypothesis/strategies.py
@@ -525,7 +525,7 @@ def sampled_from(elements):
     1 values with 10, and sampled_from((1, 10)) will shrink by trying to
     replace 10 values with 1.
     """
-    values = check_sample(elements)
+    values = check_sample(elements, 'sampled_from')
     if not values:
         return nothing()
     if len(values) == 1:
@@ -1486,13 +1486,13 @@ def do_draw(self, data):
 @defines_strategy
 def permutations(values):
     # type: (Sequence[T]) -> SearchStrategy[List[T]]
-    """Return a strategy which returns permutations of the collection
+    """Return a strategy which returns permutations of the ordered collection
     ``values``.
 
     Examples from this strategy shrink by trying to become closer to the
     original order of values.
     """
-    values = list(values)
+    values = check_sample(values, 'permutations')
     if not values:
         return builds(list)
 
@@ -1821,7 +1821,7 @@ def __init__(self, build_context, data):
     def __call__(self, values):
         if not values:
             raise IndexError('Cannot choose from empty sequence')
-        result = choice(self.data, check_sample(values))
+        result = choice(self.data, check_sample(values, 'choices'))
         with self.build_context.local():
             self.choice_count += 1
             note('Choice #%d: %r' % (self.choice_count, result))


Test patch:
diff --git a/hypothesis-python/tests/cover/test_permutations.py b/hypothesis-python/tests/cover/test_permutations.py
--- a/hypothesis-python/tests/cover/test_permutations.py
+++ b/hypothesis-python/tests/cover/test_permutations.py
@@ -18,7 +18,8 @@
 from __future__ import division, print_function, absolute_import
 
 from hypothesis import find, given
-from hypothesis.strategies import permutations
+from tests.common.utils import checks_deprecated_behaviour
+from hypothesis.strategies import data, sets, integers, permutations
 
 
 def test_can_find_non_trivial_permutation():
@@ -38,3 +39,10 @@ def test_permutation_values_are_permutations(perm):
 @given(permutations([]))
 def test_empty_permutations_are_empty(xs):
     assert xs == []
+
+
+@checks_deprecated_behaviour
+@given(data=data(), xs=sets(integers()))
+def test_non_sequence_types_are_deprecated(data, xs):
+    p = data.draw(permutations(xs))
+    assert xs == set(p)