Repository: HypothesisWorks/hypothesis

Issue:
Document tips to ensure good shrinking behaviour
Once users get beyond the basics and start defining their own domain-specific and complex strategies, it's actually pretty important that they have some heuristics for how to make their strategy shrink well.

- order of arguments is important in  `sampled_from`, `one_of`, `s1 | s2` etc
- it's good practice to pass size caps through eg `st.composite` to allow downstream control
- limit leaves in recursive strategies

I'd be tempted to mark the first point as being subject to change if we find a better system, but the rest is pretty general.

Are there any other tips I've missed?


Discussion:

Resolving patch:
diff --git a/src/hypothesis/strategies.py b/src/hypothesis/strategies.py
--- a/src/hypothesis/strategies.py
+++ b/src/hypothesis/strategies.py
@@ -167,7 +167,11 @@ def flatmap(self, f):
 @cacheable
 def nothing():
     """This strategy never successfully draws a value and will always reject on
-    an attempt to draw."""
+    an attempt to draw.
+
+    Examples from this strategy do not shrink (because there are none).
+
+    """
     return NOTHING
 
 
@@ -180,6 +184,8 @@ def just(value):
     :func:`builds(callable) <hypothesis.strategies.builds>` instead
     of ``just(callable())`` to get a fresh value each time.
 
+    Examples from this strategy do not shrink (because there is only one).
+
     """
     from hypothesis.searchstrategy.misc import JustStrategy
 
@@ -188,7 +194,12 @@ def just(value):
 
 @defines_strategy
 def none():
-    """Return a strategy which only generates None."""
+    """Return a strategy which only generates None.
+
+    Examples from this strategy do not shrink (because there is only
+    one).
+
+    """
     return just(None)
 
 
@@ -200,6 +211,17 @@ def one_of(*args):
     strategy arguments. In which case one_of(x) and one_of(\*x) are
     equivalent.
 
+    Examples from this strategy will generally shrink to ones that come from
+    strategies earlier in the list, then shrink according to behaviour of the
+    strategy that produced them. In order to get good shrinking behaviour,
+    try to put simpler strategies first. e.g. ``one_of(none(), text())`` is
+    better than ``one_of(text(), none())``.
+
+    This is especially important when using recursive strategies. e.g.
+    ``x = st.deferred(lambda: st.none() | st.tuples(x, x))`` will shrink well,
+    but ``x = st.deferred(lambda: st.tuples(x, x) | st.none())`` will shrink
+    very badly indeed.
+
     """
     if len(args) == 1 and not isinstance(args[0], SearchStrategy):
         try:
@@ -219,6 +241,9 @@ def integers(min_value=None, max_value=None):
     If min_value is not None then all values will be >= min_value. If
     max_value is not None then all values will be <= max_value
 
+    Examples from this strategy will shrink towards being positive (e.g. 1000
+    is considered simpler than -1) and then towards zero.
+
     """
 
     check_valid_bound(min_value, 'min_value')
@@ -264,7 +289,12 @@ def integers(min_value=None, max_value=None):
 @cacheable
 @defines_strategy
 def booleans():
-    """Returns a strategy which generates instances of bool."""
+    """Returns a strategy which generates instances of bool.
+
+    Examples from this strategy will shrink towards False (i.e.
+    shrinking will try to replace True with False where possible).
+
+    """
     from hypothesis.searchstrategy.misc import BoolStrategy
     return BoolStrategy()
 
@@ -286,6 +316,11 @@ def floats(
     Where not explicitly ruled out by the bounds, all of infinity, -infinity
     and NaN are possible values generated by this strategy.
 
+    Examples from this strategy have a complicated and hard to explain
+    shrinking behaviour, but it tries to improve "human readability". Finite
+    numbers will be preferred to infinity and infinity will be preferred to
+    NaN.
+
     """
 
     if allow_nan is None:
@@ -381,7 +416,12 @@ def floats(
 @cacheable
 @defines_strategy_with_reusable_values
 def complex_numbers():
-    """Returns a strategy that generates complex numbers."""
+    """Returns a strategy that generates complex numbers.
+
+    Examples from this strategy shrink by shrinking their component real
+    and imaginary parts.
+
+    """
     from hypothesis.searchstrategy.numbers import ComplexStrategy
     return ComplexStrategy(
         tuples(floats(), floats())
@@ -397,6 +437,8 @@ def tuples(*args):
     e.g. tuples(integers(), integers()) would generate a tuple of length
     two with both values an integer.
 
+    Examples from this strategy shrink by shrinking their component parts.
+
     """
     for arg in args:
         check_strategy(arg)
@@ -416,6 +458,11 @@ def sampled_from(elements):
     :class:`~python:enum.Enum` objects.  :class:`~python:enum.Flag` objects
     may also generate any combination of their members.
 
+    Examples from this strategy shrink by replacing them with values earlier in
+    the list. So e.g. sampled_from((10, 1)) will shrink by trying to replace
+    1 values with 10, and sampled_from((1, 10)) will shrink by trying to
+    replace 10 values with 1.
+
     """
     from hypothesis.searchstrategy.misc import SampledFromStrategy
     from hypothesis.internal.conjecture.utils import check_sample
@@ -459,6 +506,9 @@ def lists(
     when given a value drawn from elements. The resulting list will satisfy the
     condition that for i != j, unique_by(result[i]) != unique_by(result[j]).
 
+    Examples from this strategy shrink by trying to remove elements from the
+    list, and by shrinking each individual element of the list.
+
     """
     check_valid_sizes(min_size, average_size, max_size)
     if elements is None or (max_size is not None and max_size <= 0):
@@ -527,6 +577,9 @@ def sets(elements=None, min_size=None, average_size=None, max_size=None):
     are hashable until running the test, so you can define a strategy
     for sets of an unhashable type but it will fail at test time.
 
+    Examples from this strategy shrink by trying to remove elements from the
+    set, and by shrinking each individual element of the set.
+
     """
     return lists(
         elements=elements, min_size=min_size, average_size=average_size,
@@ -587,6 +640,9 @@ def fixed_dictionaries(mapping):
     instance of OrderedDict the keys will also be in the same order,
     otherwise the order is arbitrary.
 
+    Examples from this strategy shrink by shrinking each individual value in
+    the generated dictionary.
+
     """
     from hypothesis.searchstrategy.collections import FixedKeysDictStrategy
     check_type(dict, mapping, 'mapping')
@@ -606,6 +662,9 @@ def dictionaries(
 
     The size parameters have the same interpretation as for lists.
 
+    Examples from this strategy shrink by trying to remove keys from the
+    generated dictionary, and by shrinking each generated key and value.
+
     """
     check_valid_sizes(min_size, average_size, max_size)
     if max_size == 0:
@@ -628,6 +687,8 @@ def streaming(elements):
     The result is iterable (the iterator will never terminate) and
     indexable.
 
+    Examples from this strategy shrink by trying to shrink each value drawn.
+
     .. deprecated:: 3.15.0
         Use :func:`data() <hypothesis.strategies.data>` instead.
 
@@ -672,6 +733,8 @@ def characters(whitelist_categories=None, blacklist_categories=None,
     will be not be produced. Any overlap between ``whitelist_characters`` and
     ``blacklist_characters`` will raise an exception.
 
+    Examples from this strategy shrink towards smaller codepoints.
+
     """
     if (
         min_codepoint is not None and max_codepoint is not None and
@@ -732,6 +795,9 @@ def text(
 
     min_size, max_size and average_size have the usual interpretations.
 
+    Examples from this strategy shrink towards shorter strings, and with the
+    characters in the text shrinking as per the alphabet strategy.
+
     """
     from hypothesis.searchstrategy.strings import StringStrategy
     if alphabet is None:
@@ -779,6 +845,9 @@ def from_regex(regex):
     string, while ``"."`` will return any string, and ``r"\\A.$"`` will return
     a single character optionally followed by a ``"\\n"``.
 
+    Examples from this strategy shrink towards shorter strings and lower
+    character values.
+
     """
     from hypothesis.searchstrategy.regex import regex_strategy
     return regex_strategy(regex)
@@ -794,6 +863,9 @@ def binary(
 
     min_size, average_size and max_size have the usual interpretations.
 
+    Examples from this strategy shrink towards smaller strings and lower byte
+    values.
+
     """
     from hypothesis.searchstrategy.strings import BinaryStringStrategy, \
         FixedSizeBytes
@@ -812,7 +884,11 @@ def binary(
 @defines_strategy
 def randoms():
     """Generates instances of Random (actually a Hypothesis specific
-    RandomWithSeed class which displays what it was initially seeded with)"""
+    RandomWithSeed class which displays what it was initially seeded with)
+
+    Examples from this strategy shrink to seeds closer to zero.
+
+    """
     from hypothesis.searchstrategy.misc import RandomStrategy
     return RandomStrategy(integers())
 
@@ -839,6 +915,8 @@ def random_module():
     a random number generator you should use the randoms() strategy
     which will give you one.
 
+    Examples from these strategy shrink to seeds closer to zero.
+
     """
     from hypothesis.control import cleanup
     import random
@@ -870,6 +948,9 @@ def builds(target, *args, **kwargs):
     value :const:`hypothesis.infer` as a keyword argument to
     builds, instead of a strategy for that argument to ``target``.
 
+    Examples from this strategy shrink by shrinking the argument values to
+    the target.
+
     """
     if infer in args:
         # Avoid an implementation nightmare juggling tuples and worse things
@@ -1037,6 +1118,9 @@ def fractions(min_value=None, max_value=None, max_denominator=None):
     values is no greater than max_denominator. Note that max_denominator must
     be None or a positive integer.
 
+    Examples from this strategy shrink towards smaller denominators, then
+    closer to zero.
+
     """
     min_value = try_convert(Fraction, min_value, 'min_value')
     max_value = try_convert(Fraction, max_value, 'max_value')
@@ -1140,6 +1224,9 @@ def decimals(min_value=None, max_value=None,
     If ``places`` is not None, all finite values drawn from the strategy will
     have that number of digits after the decimal place.
 
+    Examples from this strategy do not have a well defined shrink order but
+    try to maximize human readability when shrinking.
+
     """
     # Convert min_value and max_value to Decimal values, and validate args
     check_valid_integer(places)
@@ -1232,6 +1319,10 @@ def recursive(base, extend, max_leaves=100):
     So e.g. ``False``, ``[True]``, ``[False, []]``, and ``[[[[True]]]]`` are
     all valid values to be drawn from that strategy.
 
+    Examples from this strategy shrink by trying to reduce the amount of
+    recursion and by shrinking according to the shrinking behaviour of base
+    and the result of extend.
+
     """
 
     from hypothesis.searchstrategy.recursive import RecursiveStrategy
@@ -1241,7 +1332,12 @@ def recursive(base, extend, max_leaves=100):
 @defines_strategy
 def permutations(values):
     """Return a strategy which returns permutations of the collection
-    ``values``."""
+    ``values``.
+
+    Examples from this strategy shrink by trying to become closer to the
+    original order of values.
+
+    """
     from hypothesis.internal.conjecture.utils import integer_range
 
     values = list(values)
@@ -1297,6 +1393,8 @@ def datetimes(
     :py:func:`sampled_from`.  Ensure that simple values such as None or UTC
     are at the beginning of the list for proper minimisation.
 
+    Examples from this strategy shrink towards midnight on January 1st 2000.
+
     """
     # Why must bounds be naive?  In principle, we could also write a strategy
     # that took aware bounds, but the API and validation is much harder.
@@ -1335,7 +1433,11 @@ def dates(
     min_value=dt.date.min, max_value=dt.date.max,
     min_date=None, max_date=None,
 ):
-    """A strategy for dates between ``min_date`` and ``max_date``."""
+    """A strategy for dates between ``min_date`` and ``max_date``.
+
+    Examples from this strategy shrink towards January 1st 2000.
+
+    """
     from hypothesis.searchstrategy.datetime import DateStrategy
 
     check_type(dt.date, min_value, 'min_value')
@@ -1359,6 +1461,9 @@ def times(
 
     The ``timezones`` argument is handled as for :py:func:`datetimes`.
 
+    Examples from this strategy shrink towards midnight, with the timezone
+    component shrinking as for the strategy that provided it.
+
     """
     check_type(dt.time, min_value, 'min_value')
     check_type(dt.time, max_value, 'max_value')
@@ -1382,7 +1487,11 @@ def timedeltas(
     min_value=dt.timedelta.min, max_value=dt.timedelta.max,
     min_delta=None, max_delta=None
 ):
-    """A strategy for timedeltas between ``min_value`` and ``max_value``."""
+    """A strategy for timedeltas between ``min_value`` and ``max_value``.
+
+    Examples from this strategy shrink towards zero.
+
+    """
     from hypothesis.searchstrategy.datetime import TimedeltaStrategy
 
     check_type(dt.timedelta, min_value, 'min_value')
@@ -1402,6 +1511,9 @@ def composite(f):
     :ref:`the full documentation for more details <composite-strategies>`
     about how to use this function.
 
+    Examples from this strategy shrink by shrinking the output of each draw
+    call.
+
     """
 
     from hypothesis.internal.reflection import define_function_signature
@@ -1456,6 +1568,8 @@ def shared(base, key=None):
     >>> x = shared(s, key="hi")
     >>> y = shared(s, key="hi")
 
+    Examples from this strategy shrink as per their base strategy.
+
     """
     from hypothesis.searchstrategy.shared import SharedStrategy
     return SharedStrategy(base, key)
@@ -1472,6 +1586,9 @@ def choices():
         Use :func:`data() <hypothesis.strategies.data>` with
         :func:`sampled_from() <hypothesis.strategies.sampled_from>` instead.
 
+    Examples from this strategy shrink by making each choice function return
+    an earlier value in the sequence passed to it.
+
     """
     from hypothesis.control import note, current_build_context
     from hypothesis.internal.conjecture.utils import choice, check_sample
@@ -1524,6 +1641,8 @@ def uuids(version=None):
     All returned values from this will be unique, so e.g. if you do
     ``lists(uuids())`` the resulting list will never contain duplicates.
 
+    Examples from this strategy don't have any meaningful shrink order.
+
     """
     from uuid import UUID
     if version not in (None, 1, 2, 3, 4, 5):
@@ -1545,6 +1664,8 @@ def runner(default=not_set):
     If there is no current test runner and a default is provided, return
     that default. If no default is provided, raises InvalidArgument.
 
+    Examples from this strategy do not shrink (because there is only one).
+
     """
     class RunnerStrategy(SearchStrategy):
 
@@ -1575,6 +1696,9 @@ def data():
     See :ref:`the rest of the documentation <interactive-draw>` for more
     complete information.
 
+    Examples from this strategy do not shrink (because there is only one),
+    but the result of calls to each draw() call shrink as they normally would.
+
     """
     from hypothesis.control import note
 
@@ -1684,6 +1808,9 @@ def deferred(definition):
     >>> b.example()
     (False, True)
 
+    Examples from this strategy shrink as they normally would from the strategy
+    returned by the definition.
+
     """
     from hypothesis.searchstrategy.deferred import DeferredStrategy
     return DeferredStrategy(definition)


Test patch: