Repository: HypothesisWorks/hypothesis

Issue:
Add new `--equivalent` ghostwriter which allows same-typed exceptions
The Ghostwriter often writes over-broad input strategies, which we accept on the basis that the user should be responsible for narrowing this down.

However, for the `--equivalent` property, we might (optionally) be OK with tested functions raising an exception - so long as they raise *the same exception type*.  I'd implement this with an additional argument in the Python API, and a special value which can be passed via `--except=<some special value>` on the CLI.  (still need to think of a good name for that)


Discussion:

Resolving patch:
diff --git a/hypothesis-python/src/hypothesis/extra/cli.py b/hypothesis-python/src/hypothesis/extra/cli.py
--- a/hypothesis-python/src/hypothesis/extra/cli.py
+++ b/hypothesis-python/src/hypothesis/extra/cli.py
@@ -182,8 +182,24 @@ def codemod(path):
         flag_value="equivalent",
         help="very useful when optimising or refactoring code",
     )
-    @click.option("--idempotent", "writer", flag_value="idempotent")
-    @click.option("--binary-op", "writer", flag_value="binary_operation")
+    @click.option(
+        "--errors-equivalent",
+        "writer",
+        flag_value="errors-equivalent",
+        help="--equivalent, but also allows consistent errors",
+    )
+    @click.option(
+        "--idempotent",
+        "writer",
+        flag_value="idempotent",
+        help="check that f(x) == f(f(x))",
+    )
+    @click.option(
+        "--binary-op",
+        "writer",
+        flag_value="binary_operation",
+        help="associativity, commutativity, identity element",
+    )
     # Note: we deliberately omit a --ufunc flag, because the magic()
     # detection of ufuncs is both precise and complete.
     @click.option(
@@ -218,14 +234,18 @@ def write(func, writer, except_, style):  # noqa: D301  # \b disables autowrap
         # NOTE: if you want to call this function from Python, look instead at the
         # ``hypothesis.extra.ghostwriter`` module.  Click-decorated functions have
         # a different calling convention, and raise SystemExit instead of returning.
+        kwargs = {"except_": except_ or (), "style": style}
         if writer is None:
             writer = "magic"
         elif writer == "idempotent" and len(func) > 1:
             raise click.UsageError("Test functions for idempotence one at a time.")
         elif writer == "roundtrip" and len(func) == 1:
             writer = "idempotent"
-        elif writer == "equivalent" and len(func) == 1:
+        elif "equivalent" in writer and len(func) == 1:
             writer = "fuzz"
+        if writer == "errors-equivalent":
+            writer = "equivalent"
+            kwargs["allow_same_errors"] = True
 
         try:
             from hypothesis.extra import ghostwriter
@@ -233,7 +253,7 @@ def write(func, writer, except_, style):  # noqa: D301  # \b disables autowrap
             sys.stderr.write(MESSAGE.format("black"))
             sys.exit(1)
 
-        code = getattr(ghostwriter, writer)(*func, except_=except_ or (), style=style)
+        code = getattr(ghostwriter, writer)(*func, **kwargs)
         try:
             from rich.console import Console
             from rich.syntax import Syntax
diff --git a/hypothesis-python/src/hypothesis/extra/ghostwriter.py b/hypothesis-python/src/hypothesis/extra/ghostwriter.py
--- a/hypothesis-python/src/hypothesis/extra/ghostwriter.py
+++ b/hypothesis-python/src/hypothesis/extra/ghostwriter.py
@@ -42,8 +42,9 @@
     Options:
       --roundtrip                start by testing write/read or encode/decode!
       --equivalent               very useful when optimising or refactoring code
-      --idempotent
-      --binary-op
+      --errors-equivalent        --equivalent, but also allows consistent errors
+      --idempotent               check that f(x) == f(f(x))
+      --binary-op                associativity, commutativity, identity element
       --style [pytest|unittest]  pytest-style function, or unittest-style method?
       -e, --except OBJ_NAME      dotted name of exception(s) to ignore
       -h, --help                 Show this message and exit.
@@ -126,14 +127,15 @@ def test_{test_kind}_{func_name}({arg_names}):
 {test_body}
 """
 
-SUPPRESS_BLOCK = """\
+SUPPRESS_BLOCK = """
 try:
 {test_body}
 except {exceptions}:
     reject()
-""".rstrip()
+""".strip()
 
 Except = Union[Type[Exception], Tuple[Type[Exception], ...]]
+ImportSet = Set[Union[str, Tuple[str, str]]]
 RE_TYPES = (type(re.compile(".")), type(re.match(".", "abc")))
 _quietly_settings = settings(
     database=None,
@@ -171,6 +173,22 @@ def _check_except(except_: Except) -> Tuple[Type[Exception], ...]:
     return (except_,)
 
 
+def _exception_string(except_: Tuple[Type[Exception], ...]) -> Tuple[ImportSet, str]:
+    if not except_:
+        return set(), ""
+    exceptions = []
+    imports: ImportSet = set()
+    for ex in _dedupe_exceptions(except_):
+        if ex.__qualname__ in dir(builtins):
+            exceptions.append(ex.__qualname__)
+        else:
+            imports.add(ex.__module__)
+            exceptions.append(_get_qualname(ex, include_module=True))
+    return imports, (
+        "(" + ", ".join(exceptions) + ")" if len(exceptions) > 1 else exceptions[0]
+    )
+
+
 def _check_style(style: str) -> None:
     if style not in ("pytest", "unittest"):
         raise InvalidArgument(f"Valid styles are 'pytest' or 'unittest', got {style!r}")
@@ -623,7 +641,7 @@ def _make_test_body(
     assertions: str = "",
     style: str,
     given_strategies: Optional[Mapping[str, Union[str, st.SearchStrategy]]] = None,
-) -> Tuple[Set[Union[str, Tuple[str, str]]], str]:
+) -> Tuple[ImportSet, str]:
     # A set of modules to import - we might add to this later.  The import code
     # is written later, so we can have one import section for multiple magic()
     # test functions.
@@ -641,20 +659,13 @@ def _make_test_body(
 
     if except_:
         # Convert to strings, either builtin names or qualified names.
-        exceptions = []
-        for ex in _dedupe_exceptions(except_):
-            if ex.__qualname__ in dir(builtins):
-                exceptions.append(ex.__qualname__)
-            else:
-                imports.add(ex.__module__)
-                exceptions.append(_get_qualname(ex, include_module=True))
+        imp, exc_string = _exception_string(except_)
+        imports.update(imp)
         # And finally indent the existing test body into a try-except block
         # which catches these exceptions and calls `hypothesis.reject()`.
         test_body = SUPPRESS_BLOCK.format(
             test_body=indent(test_body, prefix="    "),
-            exceptions="(" + ", ".join(exceptions) + ")"
-            if len(exceptions) > 1
-            else exceptions[0],
+            exceptions=exc_string,
         )
 
     if assertions:
@@ -682,7 +693,7 @@ def _make_test_body(
     return imports, body
 
 
-def _make_test(imports: Set[Union[str, Tuple[str, str]]], body: str) -> str:
+def _make_test(imports: ImportSet, body: str) -> str:
     # Discarding "builtins." and "__main__" probably isn't particularly useful
     # for user code, but important for making a good impression in demos.
     body = body.replace("builtins.", "").replace("__main__.", "")
@@ -1067,11 +1078,72 @@ def _make_equiv_body(funcs, except_, style):
     )
 
 
-def equivalent(*funcs: Callable, except_: Except = (), style: str = "pytest") -> str:
+EQUIV_FIRST_BLOCK = """
+try:
+{}
+    exc_type = None
+    target(1, label="input was valid")
+{}except Exception as exc:
+    exc_type = type(exc)
+""".strip()
+
+EQUIV_CHECK_BLOCK = """
+if exc_type:
+    with {ctx}(exc_type):
+{check_raises}
+else:
+{call}
+{compare}
+""".rstrip()
+
+
+def _make_equiv_errors_body(funcs, except_, style):
+    var_names = [f"result_{f.__name__}" for f in funcs]
+    if len(set(var_names)) < len(var_names):
+        var_names = [f"result_{i}_{ f.__name__}" for i, f in enumerate(funcs)]
+
+    first, *rest = funcs
+    first_call = _write_call(first, assign=var_names[0], except_=except_)
+    extra_imports, suppress = _exception_string(except_)
+    extra_imports.add(("hypothesis", "target"))
+    catch = f"except {suppress}:\n    reject()\n" if suppress else ""
+    test_lines = [EQUIV_FIRST_BLOCK.format(indent(first_call, prefix="    "), catch)]
+
+    for vname, f in zip(var_names[1:], rest):
+        if style == "pytest":
+            ctx = "pytest.raises"
+            extra_imports.add("pytest")
+        else:
+            assert style == "unittest"
+            ctx = "self.assertRaises"
+        block = EQUIV_CHECK_BLOCK.format(
+            ctx=ctx,
+            check_raises=indent(_write_call(f, except_=()), "        "),
+            call=indent(_write_call(f, assign=vname, except_=()), "    "),
+            compare=indent(_assert_eq(style, var_names[0], vname), "    "),
+        )
+        test_lines.append(block)
+
+    imports, source_code = _make_test_body(
+        *funcs,
+        test_body="\n".join(test_lines),
+        except_=(),
+        ghost="equivalent",
+        style=style,
+    )
+    return imports | extra_imports, source_code
+
+
+def equivalent(
+    *funcs: Callable,
+    allow_same_errors: bool = False,
+    except_: Except = (),
+    style: str = "pytest",
+) -> str:
     """Write source code for a property-based test of ``funcs``.
 
-    The resulting test checks that calling each of the functions gives
-    the same result.  This can be used as a classic 'oracle', such as testing
+    The resulting test checks that calling each of the functions returns
+    an equal value.  This can be used as a classic 'oracle', such as testing
     a fast sorting algorithm against the :func:`python:sorted` builtin, or
     for differential testing where none of the compared functions are fully
     trusted but any difference indicates a bug (e.g. running a function on
@@ -1080,15 +1152,25 @@ def equivalent(*funcs: Callable, except_: Except = (), style: str = "pytest") ->
     The functions should have reasonably similar signatures, as only the
     common parameters will be passed the same arguments - any other parameters
     will be allowed to vary.
+
+    If allow_same_errors is True, then the test will pass if calling each of
+    the functions returns an equal value, *or* if the first function raises an
+    exception and each of the others raises an exception of the same type.
+    This relaxed mode can be useful for code synthesis projects.
     """
     if len(funcs) < 2:
         raise InvalidArgument("Need at least two functions to compare.")
     for i, f in enumerate(funcs):
         if not callable(f):
             raise InvalidArgument(f"Got non-callable funcs[{i}]={f!r}")
+    check_type(bool, allow_same_errors, "allow_same_errors")
     except_ = _check_except(except_)
     _check_style(style)
-    return _make_test(*_make_equiv_body(funcs, except_, style))
+    if allow_same_errors and not any(issubclass(Exception, ex) for ex in except_):
+        imports, source_code = _make_equiv_errors_body(funcs, except_, style)
+    else:
+        imports, source_code = _make_equiv_body(funcs, except_, style)
+    return _make_test(imports, source_code)
 
 
 X = TypeVar("X")
@@ -1165,7 +1247,7 @@ def _make_binop_body(
     distributes_over: Optional[Callable[[X, X], X]] = None,
     except_: Tuple[Type[Exception], ...],
     style: str,
-) -> Tuple[Set[Union[str, Tuple[str, str]]], str]:
+) -> Tuple[ImportSet, str]:
     strategies = _get_strategies(func)
     operands, b = (strategies.pop(p) for p in list(_get_params(func))[:2])
     if repr(operands) != repr(b):


Test patch:
diff --git a/hypothesis-python/tests/ghostwriter/recorded/sorted_self_error_equivalent_1error.txt b/hypothesis-python/tests/ghostwriter/recorded/sorted_self_error_equivalent_1error.txt
new file mode 100644
--- /dev/null
+++ b/hypothesis-python/tests/ghostwriter/recorded/sorted_self_error_equivalent_1error.txt
@@ -0,0 +1,28 @@
+# This test code was written by the `hypothesis.extra.ghostwriter` module
+# and is provided under the Creative Commons Zero public domain dedication.
+
+import pytest
+from hypothesis import given, reject, strategies as st, target
+
+
+@given(
+    iterable=st.one_of(st.iterables(st.integers()), st.iterables(st.text())),
+    key=st.none(),
+    reverse=st.booleans(),
+)
+def test_equivalent_sorted_sorted(iterable, key, reverse):
+    try:
+        result_0_sorted = sorted(iterable, key=key, reverse=reverse)
+        exc_type = None
+        target(1, label="input was valid")
+    except ValueError:
+        reject()
+    except Exception as exc:
+        exc_type = type(exc)
+
+    if exc_type:
+        with pytest.raises(exc_type):
+            sorted(iterable, key=key, reverse=reverse)
+    else:
+        result_1_sorted = sorted(iterable, key=key, reverse=reverse)
+        assert result_0_sorted == result_1_sorted, (result_0_sorted, result_1_sorted)
diff --git a/hypothesis-python/tests/ghostwriter/recorded/sorted_self_error_equivalent_2error_unittest.txt b/hypothesis-python/tests/ghostwriter/recorded/sorted_self_error_equivalent_2error_unittest.txt
new file mode 100644
--- /dev/null
+++ b/hypothesis-python/tests/ghostwriter/recorded/sorted_self_error_equivalent_2error_unittest.txt
@@ -0,0 +1,29 @@
+# This test code was written by the `hypothesis.extra.ghostwriter` module
+# and is provided under the Creative Commons Zero public domain dedication.
+
+import unittest
+from hypothesis import given, reject, strategies as st, target
+
+
+class TestEquivalentSortedSorted(unittest.TestCase):
+    @given(
+        iterable=st.one_of(st.iterables(st.integers()), st.iterables(st.text())),
+        key=st.none(),
+        reverse=st.booleans(),
+    )
+    def test_equivalent_sorted_sorted(self, iterable, key, reverse):
+        try:
+            result_0_sorted = sorted(iterable, key=key, reverse=reverse)
+            exc_type = None
+            target(1, label="input was valid")
+        except (TypeError, ValueError):
+            reject()
+        except Exception as exc:
+            exc_type = type(exc)
+
+        if exc_type:
+            with self.assertRaises(exc_type):
+                sorted(iterable, key=key, reverse=reverse)
+        else:
+            result_1_sorted = sorted(iterable, key=key, reverse=reverse)
+            self.assertEqual(result_0_sorted, result_1_sorted)
diff --git a/hypothesis-python/tests/ghostwriter/recorded/sorted_self_error_equivalent_simple.txt b/hypothesis-python/tests/ghostwriter/recorded/sorted_self_error_equivalent_simple.txt
new file mode 100644
--- /dev/null
+++ b/hypothesis-python/tests/ghostwriter/recorded/sorted_self_error_equivalent_simple.txt
@@ -0,0 +1,26 @@
+# This test code was written by the `hypothesis.extra.ghostwriter` module
+# and is provided under the Creative Commons Zero public domain dedication.
+
+import pytest
+from hypothesis import given, strategies as st, target
+
+
+@given(
+    iterable=st.one_of(st.iterables(st.integers()), st.iterables(st.text())),
+    key=st.none(),
+    reverse=st.booleans(),
+)
+def test_equivalent_sorted_sorted(iterable, key, reverse):
+    try:
+        result_0_sorted = sorted(iterable, key=key, reverse=reverse)
+        exc_type = None
+        target(1, label="input was valid")
+    except Exception as exc:
+        exc_type = type(exc)
+
+    if exc_type:
+        with pytest.raises(exc_type):
+            sorted(iterable, key=key, reverse=reverse)
+    else:
+        result_1_sorted = sorted(iterable, key=key, reverse=reverse)
+        assert result_0_sorted == result_1_sorted, (result_0_sorted, result_1_sorted)
diff --git a/hypothesis-python/tests/ghostwriter/recorded/sorted_self_error_equivalent_threefuncs.txt b/hypothesis-python/tests/ghostwriter/recorded/sorted_self_error_equivalent_threefuncs.txt
new file mode 100644
--- /dev/null
+++ b/hypothesis-python/tests/ghostwriter/recorded/sorted_self_error_equivalent_threefuncs.txt
@@ -0,0 +1,33 @@
+# This test code was written by the `hypothesis.extra.ghostwriter` module
+# and is provided under the Creative Commons Zero public domain dedication.
+
+import pytest
+from hypothesis import given, strategies as st, target
+
+
+@given(
+    iterable=st.one_of(st.iterables(st.integers()), st.iterables(st.text())),
+    key=st.none(),
+    reverse=st.booleans(),
+)
+def test_equivalent_sorted_sorted_sorted(iterable, key, reverse):
+    try:
+        result_0_sorted = sorted(iterable, key=key, reverse=reverse)
+        exc_type = None
+        target(1, label="input was valid")
+    except Exception as exc:
+        exc_type = type(exc)
+
+    if exc_type:
+        with pytest.raises(exc_type):
+            sorted(iterable, key=key, reverse=reverse)
+    else:
+        result_1_sorted = sorted(iterable, key=key, reverse=reverse)
+        assert result_0_sorted == result_1_sorted, (result_0_sorted, result_1_sorted)
+
+    if exc_type:
+        with pytest.raises(exc_type):
+            sorted(iterable, key=key, reverse=reverse)
+    else:
+        result_2_sorted = sorted(iterable, key=key, reverse=reverse)
+        assert result_0_sorted == result_2_sorted, (result_0_sorted, result_2_sorted)
diff --git a/hypothesis-python/tests/ghostwriter/test_expected_output.py b/hypothesis-python/tests/ghostwriter/test_expected_output.py
--- a/hypothesis-python/tests/ghostwriter/test_expected_output.py
+++ b/hypothesis-python/tests/ghostwriter/test_expected_output.py
@@ -149,6 +149,33 @@ def divide(a: int, b: int) -> float:
                 style="unittest",
             ),
         ),
+        (
+            "sorted_self_error_equivalent_simple",
+            ghostwriter.equivalent(sorted, sorted, allow_same_errors=True),
+        ),
+        (
+            "sorted_self_error_equivalent_threefuncs",
+            ghostwriter.equivalent(sorted, sorted, sorted, allow_same_errors=True),
+        ),
+        (
+            "sorted_self_error_equivalent_1error",
+            ghostwriter.equivalent(
+                sorted,
+                sorted,
+                allow_same_errors=True,
+                except_=ValueError,
+            ),
+        ),
+        (
+            "sorted_self_error_equivalent_2error_unittest",
+            ghostwriter.equivalent(
+                sorted,
+                sorted,
+                allow_same_errors=True,
+                except_=(TypeError, ValueError),
+                style="unittest",
+            ),
+        ),
         pytest.param(
             ("magic_builtins", ghostwriter.magic(builtins)),
             marks=[