Repository: HypothesisWorks/hypothesis

Issue:
Improve the `extra.numpy.array_dtypes` strategy
[Current docs here](https://hypothesis.readthedocs.io/en/latest/numpy.html#hypothesis.extra.numpy.array_dtypes).

When reading [the SciPy docs on structured arrays](https://docs.scipy.org/doc/numpy/user/basics.rec.html#field-titles). I discovered that array dtypes / record dtypes / structured dtypes (we should probably mention those other terms in the docs...) can have a *title* as well as a field name.  This title is *also included in the `.fields` attribute*, which I doubt much existing code is prepared to handle.  We should generate fields with titles.

I'd also like to make us much more likely to generate strings like `"f{index}"`, which are used to replace the empty string, and to permit the empty string as a name where the corresponding replacement is not otherwise in the array (which is an error).  This applies for both names and titles, depending on how they interact - having one field whose title is the name of another field would probably be interesting :smile: 

It would also be useful to support control of alignment, and of field offsets (overlapping fields are supported, and can lead to data clobbering...).

CC @rsokl for interest.


Discussion:

Resolving patch:
diff --git a/hypothesis-python/src/hypothesis/extra/numpy.py b/hypothesis-python/src/hypothesis/extra/numpy.py
--- a/hypothesis-python/src/hypothesis/extra/numpy.py
+++ b/hypothesis-python/src/hypothesis/extra/numpy.py
@@ -15,7 +15,6 @@
 
 import math
 import re
-from collections import namedtuple
 from typing import Any, NamedTuple, Sequence, Tuple, Union
 
 import numpy as np
@@ -651,6 +650,18 @@ def unicode_string_dtypes(
     return dtype_factory("U", list(range(min_len, max_len + 1)), None, endianness)
 
 
+def _no_title_is_name_of_a_titled_field(ls):
+    seen = set()
+    for title_and_name, *_ in ls:
+        if isinstance(title_and_name, tuple):
+            if seen.intersection(title_and_name):  # pragma: no cover
+                # Our per-element filters below make this as rare as possible,
+                # so it's not always covered.
+                return False
+            seen.update(title_and_name)
+    return True
+
+
 @defines_dtype_strategy
 @deprecated_posargs
 def array_dtypes(
@@ -663,19 +674,29 @@ def array_dtypes(
     """Return a strategy for generating array (compound) dtypes, with members
     drawn from the given subtype strategy."""
     order_check("size", 0, min_size, max_size)
-    # Field names must be native strings and the empty string is weird; see #1963.
-    field_names = st.text(min_size=1)
-    elements = st.tuples(field_names, subtype_strategy)
+    # The empty string is replaced by f{idx}; see #1963 for details.  Much easier to
+    # insist that field names be unique and just boost f{idx} strings manually.
+    field_names = st.integers(0, 127).map("f{}".format) | st.text(min_size=1)
+    name_titles = st.one_of(
+        field_names,
+        st.tuples(field_names, field_names).filter(lambda ns: ns[0] != ns[1]),
+    )
+    elements = st.tuples(name_titles, subtype_strategy)
     if allow_subarrays:
         elements |= st.tuples(
-            field_names, subtype_strategy, array_shapes(max_dims=2, max_side=2)
+            name_titles, subtype_strategy, array_shapes(max_dims=2, max_side=2)
         )
     return st.lists(
         elements=elements,
         min_size=min_size,
         max_size=max_size,
-        unique_by=lambda d: d[0],
-    )
+        unique_by=(
+            # Deduplicate by both name and title for efficiency before filtering.
+            # (Field names must be unique, as must titles, and no intersections)
+            lambda d: d[0] if isinstance(d[0], str) else d[0][0],
+            lambda d: d[0] if isinstance(d[0], str) else d[0][1],
+        ),
+    ).filter(_no_title_is_name_of_a_titled_field)
 
 
 @st.defines_strategy
@@ -1011,7 +1032,9 @@ def _draw_loop_dimensions(self, data, use=None):
 _SIGNATURE = r"^{}->{}$".format(_ARGUMENT_LIST, _SHAPE)
 _SIGNATURE_MULTIPLE_OUTPUT = r"^{0}->{0}$".format(_ARGUMENT_LIST)
 
-_GUfuncSig = namedtuple("_GUfuncSig", ["input_shapes", "result_shape"])
+_GUfuncSig = NamedTuple(
+    "_GUfuncSig", [("input_shapes", Tuple[Shape, ...]), ("result_shape", Shape)]
+)
 
 
 def _hypothesis_parse_gufunc_signature(signature, all_checks=True):


Test patch:
diff --git a/hypothesis-python/tests/numpy/test_gen_data.py b/hypothesis-python/tests/numpy/test_gen_data.py
--- a/hypothesis-python/tests/numpy/test_gen_data.py
+++ b/hypothesis-python/tests/numpy/test_gen_data.py
@@ -249,8 +249,12 @@ def test_minimise_array_strategy():
 
 @given(nps.array_dtypes(allow_subarrays=False))
 def test_can_turn_off_subarrays(dt):
-    for field, _ in dt.fields.values():
-        assert field.shape == ()
+    for name in dt.names:
+        assert dt.fields[name][0].shape == ()
+
+
+def test_array_dtypes_may_have_field_titles():
+    find_any(nps.array_dtypes(), lambda dt: len(dt.fields) > len(dt.names))
 
 
 @pytest.mark.parametrize("byteorder", ["<", ">"])