Repository: numpy/numpy

Issue:
Bug in masked array indexing.
This simple indexing operation

```
ma = np.ma.zeros(5000, dtype=[('symbol', 'O')])
print(ma[0])
```

fails with a

```
ValueError: Setting void-array with object members using buffer.
```

exception. While the non-masked version just works:

```
arr = np.zeros(5000, dtype=[('symbol', 'O')])
print(arr[0])
```

Numpy version `1.11.0` on python 3.5.1 on arch linux.

Structured array drops field-titles when being 'sliced' by field-names
Is the following behaviour of numpy (v1.13.1) a bug or by design?

```
>>>import numpy as np
>>>a = np.zeros((1,), dtype=[(('title 1', 'x'), '|i1'), (('title 2', 'y'), '>f4')])

>>>a.dtype.descr
[(('title 1', 'x'), '|i1'), (('title 2', 'y'), '>f4')]

>>>a[['y','x']].dtype.descr
[('y', '>f4'), ('x', '|i1')]
```

I would have expected the last expression to have returned this instead: 
`[(('title 2', 'y'), '>f4'), (('title 1', 'x'), '|i1')]`

Why are the field-titles missing on a view obtained by indexing?
It is kind of surprising, that some aspects of the structured-array dtype just get dropped.



Discussion:

Resolving patch:
diff --git a/numpy/ma/core.py b/numpy/ma/core.py
--- a/numpy/ma/core.py
+++ b/numpy/ma/core.py
@@ -1613,6 +1613,11 @@ def make_mask(m, copy=False, shrink=True, dtype=MaskType):
 
     # Make sure the input dtype is valid.
     dtype = make_mask_descr(dtype)
+    
+    # legacy boolean special case: "existence of fields implies true"
+    if isinstance(m, ndarray) and m.dtype.fields and dtype == np.bool_:
+        return np.ones(m.shape, dtype=dtype)
+
     # Fill the mask in case there are missing data; turn it into an ndarray.
     result = np.array(filled(m, True), copy=copy, dtype=dtype, subok=True)
     # Bas les masques !


Test patch:
diff --git a/numpy/core/tests/test_dtype.py b/numpy/core/tests/test_dtype.py
--- a/numpy/core/tests/test_dtype.py
+++ b/numpy/core/tests/test_dtype.py
@@ -104,6 +104,15 @@ def test_bad_param(self):
                          'formats':['i1', 'f4'],
                          'offsets':[0, 2]}, align=True)
 
+    def test_field_order_equality(self):
+        x = np.dtype({'names': ['A', 'B'], 
+                      'formats': ['i4', 'f4'], 
+                      'offsets': [0, 4]})
+        y = np.dtype({'names': ['B', 'A'], 
+                      'formats': ['f4', 'i4'], 
+                      'offsets': [4, 0]})
+        assert_equal(x == y, False)
+
 class TestRecord(object):
     def test_equivalent_record(self):
         """Test whether equivalent record dtypes hash the same."""
@@ -211,11 +220,12 @@ def test_union_struct(self):
         dt = np.dtype({'names':['f0', 'f1', 'f2'], 'formats':['<u4', '<u2', '<u2'],
                         'offsets':[4, 0, 2]}, align=True)
         assert_equal(dt.itemsize, 8)
+        # field name should not matter: assignment is by position
         dt2 = np.dtype({'names':['f2', 'f0', 'f1'],
-                        'formats':['<u2', '<u4', '<u2'],
-                        'offsets':[2, 4, 0]}, align=True)
+                        'formats':['<u4', '<u2', '<u2'],
+                        'offsets':[4, 0, 2]}, align=True)
         vals = [(0, 1, 2), (3, -1, 4)]
-        vals2 = [(2, 0, 1), (4, 3, -1)]
+        vals2 = [(0, 1, 2), (3, -1, 4)]
         a = np.array(vals, dt)
         b = np.array(vals2, dt2)
         assert_equal(a.astype(dt2), b)
diff --git a/numpy/core/tests/test_indexing.py b/numpy/core/tests/test_indexing.py
--- a/numpy/core/tests/test_indexing.py
+++ b/numpy/core/tests/test_indexing.py
@@ -106,6 +106,12 @@ def test_empty_tuple_index(self):
         a = np.array(0)
         assert_(isinstance(a[()], np.int_))
 
+    def test_void_scalar_empty_tuple(self):
+        s = np.zeros((), dtype='V4')
+        assert_equal(s[()].dtype, s.dtype)
+        assert_equal(s[()], s)
+        assert_equal(type(s[...]), np.ndarray)
+
     def test_same_kind_index_casting(self):
         # Indexes should be cast with same-kind and not safe, even if that
         # is somewhat unsafe. So test various different code paths.
diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py
--- a/numpy/core/tests/test_multiarray.py
+++ b/numpy/core/tests/test_multiarray.py
@@ -952,16 +952,13 @@ def test_casting(self):
         # Check that equality comparison works on structured arrays if
         # they are 'equiv'-castable
         a = np.array([(5, 42), (10, 1)], dtype=[('a', '>i4'), ('b', '<f8')])
-        b = np.array([(42, 5), (1, 10)], dtype=[('b', '>f8'), ('a', '<i4')])
+        b = np.array([(5, 42), (10, 1)], dtype=[('a', '<i4'), ('b', '>f8')])
         assert_(np.can_cast(a.dtype, b.dtype, casting='equiv'))
         assert_equal(a == b, [True, True])
 
-        # Check that 'equiv' casting can reorder fields and change byte
-        # order
-        # New in 1.12: This behavior changes in 1.13, test for dep warning
+        # Check that 'equiv' casting can change byte order
         assert_(np.can_cast(a.dtype, b.dtype, casting='equiv'))
-        with assert_warns(FutureWarning):
-            c = a.astype(b.dtype, casting='equiv')
+        c = a.astype(b.dtype, casting='equiv')
         assert_equal(a == c, [True, True])
 
         # Check that 'safe' casting can change byte order and up-cast
@@ -1096,6 +1093,54 @@ def test_base_attr(self):
         b = a[0]
         assert_(b.base is a)
 
+    def test_assignment(self):
+        def testassign(arr, v):
+            c = arr.copy()
+            c[0] = v  # assign using setitem
+            c[1:] = v # assign using "dtype_transfer" code paths
+            return c
+
+        dt = np.dtype([('foo', 'i8'), ('bar', 'i8')])
+        arr = np.ones(2, dt)
+        v1 = np.array([(2,3)], dtype=[('foo', 'i8'), ('bar', 'i8')])
+        v2 = np.array([(2,3)], dtype=[('bar', 'i8'), ('foo', 'i8')])
+        v3 = np.array([(2,3)], dtype=[('bar', 'i8'), ('baz', 'i8')])
+        v4 = np.array([(2,)],  dtype=[('bar', 'i8')])
+        v5 = np.array([(2,3)], dtype=[('foo', 'f8'), ('bar', 'f8')])
+        w = arr.view({'names': ['bar'], 'formats': ['i8'], 'offsets': [8]})
+
+        ans = np.array([(2,3),(2,3)], dtype=dt)
+        assert_equal(testassign(arr, v1), ans)
+        assert_equal(testassign(arr, v2), ans)
+        assert_equal(testassign(arr, v3), ans)
+        assert_raises(ValueError, lambda: testassign(arr, v4))
+        assert_equal(testassign(arr, v5), ans)
+        w[:] = 4
+        assert_equal(arr, np.array([(1,4),(1,4)], dtype=dt))
+
+        # test field-reordering, assignment by position, and self-assignment
+        a = np.array([(1,2,3)],
+                     dtype=[('foo', 'i8'), ('bar', 'i8'), ('baz', 'f4')])
+        a[['foo', 'bar']] = a[['bar', 'foo']]
+        assert_equal(a[0].item(), (2,1,3))
+
+        # test that this works even for 'simple_unaligned' structs
+        # (ie, that PyArray_EquivTypes cares about field order too)
+        a = np.array([(1,2)], dtype=[('a', 'i4'), ('b', 'i4')])
+        a[['a', 'b']] = a[['b', 'a']]
+        assert_equal(a[0].item(), (2,1))
+
+    def test_structuredscalar_indexing(self):
+        # test gh-7262
+        x = np.empty(shape=1, dtype="(2)3S,(2)3U")
+        assert_equal(x[["f0","f1"]][0], x[0][["f0","f1"]])
+        assert_equal(x[0], x[0][()])
+
+    def test_multiindex_titles(self):
+        a = np.zeros(4, dtype=[(('a', 'b'), 'i'), ('c', 'i'), ('d', 'i')])
+        assert_raises(KeyError, lambda : a[['a','c']])
+        assert_raises(KeyError, lambda : a[['b','b']])
+        a[['b','c']]  # no exception
 
 class TestBool(object):
     def test_test_interning(self):
@@ -4496,23 +4541,11 @@ def test_field_names(self):
             # multiple subfields
             fn2 = func('f2')
             b[fn2] = 3
-            with suppress_warnings() as sup:
-                sup.filter(FutureWarning,
-                           "Assignment between structured arrays.*")
-                sup.filter(FutureWarning,
-                           "Numpy has detected that you .*")
-
-                assert_equal(b[['f1', 'f2']][0].tolist(), (2, 3))
-                assert_equal(b[['f2', 'f1']][0].tolist(), (3, 2))
-                assert_equal(b[['f1', 'f3']][0].tolist(), (2, (1,)))
-                # view of subfield view/copy
-                assert_equal(b[['f1', 'f2']][0].view(('i4', 2)).tolist(),
-                             (2, 3))
-                assert_equal(b[['f2', 'f1']][0].view(('i4', 2)).tolist(),
-                             (3, 2))
-                view_dtype = [('f1', 'i4'), ('f3', [('', 'i4')])]
-                assert_equal(b[['f1', 'f3']][0].view(view_dtype).tolist(),
-                             (2, (1,)))
+
+            assert_equal(b[['f1', 'f2']][0].tolist(), (2, 3))
+            assert_equal(b[['f2', 'f1']][0].tolist(), (3, 2))
+            assert_equal(b[['f1', 'f3']][0].tolist(), (2, (1,)))
+
         # non-ascii unicode field indexing is well behaved
         if not is_py3:
             raise SkipTest('non ascii unicode field indexing skipped; '
@@ -4521,54 +4554,6 @@ def test_field_names(self):
             assert_raises(ValueError, a.__setitem__, u'\u03e0', 1)
             assert_raises(ValueError, a.__getitem__, u'\u03e0')
 
-    def test_field_names_deprecation(self):
-
-        def collect_warnings(f, *args, **kwargs):
-            with warnings.catch_warnings(record=True) as log:
-                warnings.simplefilter("always")
-                f(*args, **kwargs)
-            return [w.category for w in log]
-
-        a = np.zeros((1,), dtype=[('f1', 'i4'),
-                                  ('f2', 'i4'),
-                                  ('f3', [('sf1', 'i4')])])
-        a['f1'][0] = 1
-        a['f2'][0] = 2
-        a['f3'][0] = (3,)
-        b = np.zeros((1,), dtype=[('f1', 'i4'),
-                                  ('f2', 'i4'),
-                                  ('f3', [('sf1', 'i4')])])
-        b['f1'][0] = 1
-        b['f2'][0] = 2
-        b['f3'][0] = (3,)
-
-        # All the different functions raise a warning, but not an error
-        assert_equal(collect_warnings(a[['f1', 'f2']].__setitem__, 0, (10, 20)),
-                     [FutureWarning])
-        # For <=1.12 a is not modified, but it will be in 1.13
-        assert_equal(a, b)
-
-        # Views also warn
-        subset = a[['f1', 'f2']]
-        subset_view = subset.view()
-        assert_equal(collect_warnings(subset_view['f1'].__setitem__, 0, 10),
-                     [FutureWarning])
-        # But the write goes through:
-        assert_equal(subset['f1'][0], 10)
-        # Only one warning per multiple field indexing, though (even if there
-        # are multiple views involved):
-        assert_equal(collect_warnings(subset['f1'].__setitem__, 0, 10), [])
-
-        # make sure views of a multi-field index warn too
-        c = np.zeros(3, dtype='i8,i8,i8')
-        assert_equal(collect_warnings(c[['f0', 'f2']].view, 'i8,i8'),
-                     [FutureWarning])
-
-        # make sure assignment using a different dtype warns
-        a = np.zeros(2, dtype=[('a', 'i4'), ('b', 'i4')])
-        b = np.zeros(2, dtype=[('b', 'i4'), ('a', 'i4')])
-        assert_equal(collect_warnings(a.__setitem__, (), b), [FutureWarning])
-
     def test_record_hash(self):
         a = np.array([(1, 2), (1, 2)], dtype='i1,i2')
         a.flags.writeable = False
diff --git a/numpy/core/tests/test_nditer.py b/numpy/core/tests/test_nditer.py
--- a/numpy/core/tests/test_nditer.py
+++ b/numpy/core/tests/test_nditer.py
@@ -1816,100 +1816,45 @@ def test_iter_buffered_cast_structured_type():
     if HAS_REFCOUNT:
         assert_equal(sys.getrefcount(a[0]), rc)
 
-    # struct type -> simple (takes the first value)
-    sdt = [('a', 'f4'), ('b', 'i8'), ('d', 'O')]
-    a = np.array([(5.5, 7, 'test'), (8, 10, 11)], dtype=sdt)
+    # single-field struct type -> simple
+    sdt = [('a', 'f4')]
+    a = np.array([(5.5,), (8,)], dtype=sdt)
     i = nditer(a, ['buffered', 'refs_ok'], ['readonly'],
                     casting='unsafe',
                     op_dtypes='i4')
     assert_equal([x_[()] for x_ in i], [5, 8])
 
+    # make sure multi-field struct type -> simple doesn't work
+    sdt = [('a', 'f4'), ('b', 'i8'), ('d', 'O')]
+    a = np.array([(5.5, 7, 'test'), (8, 10, 11)], dtype=sdt)
+    assert_raises(ValueError, lambda: (
+        nditer(a, ['buffered', 'refs_ok'], ['readonly'],
+               casting='unsafe',
+               op_dtypes='i4')))
+
     # struct type -> struct type (field-wise copy)
     sdt1 = [('a', 'f4'), ('b', 'i8'), ('d', 'O')]
     sdt2 = [('d', 'u2'), ('a', 'O'), ('b', 'f8')]
     a = np.array([(1, 2, 3), (4, 5, 6)], dtype=sdt1)
-    # New in 1.12: This behavior changes in 1.13, test for dep warning
-    with assert_warns(FutureWarning):
-        i = nditer(a, ['buffered', 'refs_ok'], ['readonly'],
-                        casting='unsafe',
-                        op_dtypes=sdt2)
+    i = nditer(a, ['buffered', 'refs_ok'], ['readonly'],
+                    casting='unsafe',
+                    op_dtypes=sdt2)
     assert_equal(i[0].dtype, np.dtype(sdt2))
     assert_equal([np.array(x_) for x_ in i],
-                 [np.array((3, 1, 2), dtype=sdt2),
-                  np.array((6, 4, 5), dtype=sdt2)])
+                 [np.array((1, 2, 3), dtype=sdt2),
+                  np.array((4, 5, 6), dtype=sdt2)])
 
-    # struct type -> struct type (field gets discarded)
+    # make sure struct type -> struct type with different
+    # number of fields fails
     sdt1 = [('a', 'f4'), ('b', 'i8'), ('d', 'O')]
     sdt2 = [('b', 'O'), ('a', 'f8')]
     a = np.array([(1, 2, 3), (4, 5, 6)], dtype=sdt1)
-    # New in 1.12: This behavior changes in 1.13, test for dep warning
-    with assert_warns(FutureWarning):
-        i = nditer(a, ['buffered', 'refs_ok'], ['readwrite'],
-                        casting='unsafe',
-                        op_dtypes=sdt2)
-    assert_equal(i[0].dtype, np.dtype(sdt2))
-    vals = []
-    for x in i:
-        vals.append(np.array(x))
-        x['a'] = x['b']+3
-    assert_equal(vals, [np.array((2, 1), dtype=sdt2),
-                     np.array((5, 4), dtype=sdt2)])
-    assert_equal(a, np.array([(5, 2, None), (8, 5, None)], dtype=sdt1))
-
-    # struct type -> struct type (structured field gets discarded)
-    sdt1 = [('a', 'f4'), ('b', 'i8'), ('d', [('a', 'i2'), ('b', 'i4')])]
-    sdt2 = [('b', 'O'), ('a', 'f8')]
-    a = np.array([(1, 2, (0, 9)), (4, 5, (20, 21))], dtype=sdt1)
-    # New in 1.12: This behavior changes in 1.13, test for dep warning
-    with assert_warns(FutureWarning):
-        i = nditer(a, ['buffered', 'refs_ok'], ['readwrite'],
-                        casting='unsafe',
-                        op_dtypes=sdt2)
-    assert_equal(i[0].dtype, np.dtype(sdt2))
-    vals = []
-    for x in i:
-        vals.append(np.array(x))
-        x['a'] = x['b']+3
-    assert_equal(vals, [np.array((2, 1), dtype=sdt2),
-                     np.array((5, 4), dtype=sdt2)])
-    assert_equal(a, np.array([(5, 2, (0, 0)), (8, 5, (0, 0))], dtype=sdt1))
-
-    # struct type -> struct type (structured field w/ ref gets discarded)
-    sdt1 = [('a', 'f4'), ('b', 'i8'), ('d', [('a', 'i2'), ('b', 'O')])]
-    sdt2 = [('b', 'O'), ('a', 'f8')]
-    a = np.array([(1, 2, (0, 9)), (4, 5, (20, 21))], dtype=sdt1)
-    # New in 1.12: This behavior changes in 1.13, test for dep warning
-    with assert_warns(FutureWarning):
-        i = nditer(a, ['buffered', 'refs_ok'], ['readwrite'],
-                        casting='unsafe',
-                        op_dtypes=sdt2)
-    assert_equal(i[0].dtype, np.dtype(sdt2))
-    vals = []
-    for x in i:
-        vals.append(np.array(x))
-        x['a'] = x['b']+3
-    assert_equal(vals, [np.array((2, 1), dtype=sdt2),
-                     np.array((5, 4), dtype=sdt2)])
-    assert_equal(a, np.array([(5, 2, (0, None)), (8, 5, (0, None))], dtype=sdt1))
-
-    # struct type -> struct type back (structured field w/ ref gets discarded)
-    sdt1 = [('b', 'O'), ('a', 'f8')]
-    sdt2 = [('a', 'f4'), ('b', 'i8'), ('d', [('a', 'i2'), ('b', 'O')])]
-    a = np.array([(1, 2), (4, 5)], dtype=sdt1)
-    # New in 1.12: This behavior changes in 1.13, test for dep warning
-    with assert_warns(FutureWarning):
-        i = nditer(a, ['buffered', 'refs_ok'], ['readwrite'],
-                        casting='unsafe',
-                        op_dtypes=sdt2)
-    assert_equal(i[0].dtype, np.dtype(sdt2))
-    vals = []
-    for x in i:
-        vals.append(np.array(x))
-        assert_equal(x['d'], np.array((0, None), dtype=[('a', 'i2'), ('b', 'O')]))
-        x['a'] = x['b']+3
-    assert_equal(vals, [np.array((2, 1, (0, None)), dtype=sdt2),
-                     np.array((5, 4, (0, None)), dtype=sdt2)])
-    assert_equal(a, np.array([(1, 4), (4, 7)], dtype=sdt1))
+
+    assert_raises(ValueError, lambda : (
+        nditer(a, ['buffered', 'refs_ok'], ['readwrite'],
+               casting='unsafe',
+               op_dtypes=sdt2)))
+
 
 def test_iter_buffered_cast_subarray():
     # Tests buffering of subarrays
diff --git a/numpy/core/tests/test_records.py b/numpy/core/tests/test_records.py
--- a/numpy/core/tests/test_records.py
+++ b/numpy/core/tests/test_records.py
@@ -153,11 +153,6 @@ def test_recarray_views(self):
         assert_equal(r['c'].dtype.type, np.record)
         assert_equal(type(r['c']), np.recarray)
 
-        # suppress deprecation warning in 1.12 (remove in 1.13)
-        with assert_warns(FutureWarning):
-            assert_equal(r[['a', 'b']].dtype.type, np.record)
-            assert_equal(type(r[['a', 'b']]), np.recarray)
-
         #and that it preserves subclasses (gh-6949)
         class C(np.recarray):
             pass
@@ -334,15 +329,6 @@ def test_nonwriteable_setfield(self):
         with assert_raises(ValueError):
             r.setfield([2,3], *r.dtype.fields['f'])
 
-    def test_out_of_order_fields(self):
-        """Ticket #1431."""
-        # this test will be invalid in 1.13
-        # suppress deprecation warning in 1.12 (remove in 1.13)
-        with assert_warns(FutureWarning):
-            x = self.data[['col1', 'col2']]
-            y = self.data[['col2', 'col1']]
-        assert_equal(x[0][0], y[0][1])
-
     def test_pickle_1(self):
         # Issue #1529
         a = np.array([(1, [])], dtype=[('a', np.int32), ('b', np.int32, 0)])
@@ -371,8 +357,7 @@ def test_objview_record(self):
 
         # https://github.com/numpy/numpy/issues/3256
         ra = np.recarray((2,), dtype=[('x', object), ('y', float), ('z', int)])
-        with assert_warns(FutureWarning):
-            ra[['x','y']]  # TypeError?
+        ra[['x','y']]  # TypeError?
 
     def test_record_scalar_setitem(self):
         # https://github.com/numpy/numpy/issues/3561
diff --git a/numpy/lib/tests/test_io.py b/numpy/lib/tests/test_io.py
--- a/numpy/lib/tests/test_io.py
+++ b/numpy/lib/tests/test_io.py
@@ -1178,12 +1178,12 @@ def test_dtype_with_converters_and_usecols(self):
         conv = {0: int, 1: int, 2: int, 3: lambda r: dmap[r.decode()]}
         test = np.recfromcsv(TextIO(dstr,), dtype=dtyp, delimiter=',',
                              names=None, converters=conv)
-        control = np.rec.array([[1,5,-1,0], [2,8,-1,1], [3,3,-2,3]], dtype=dtyp)
+        control = np.rec.array([(1,5,-1,0), (2,8,-1,1), (3,3,-2,3)], dtype=dtyp)
         assert_equal(test, control)
         dtyp = [('e1','i4'),('e2','i4'),('n', 'i1')]
         test = np.recfromcsv(TextIO(dstr,), dtype=dtyp, delimiter=',',
                              usecols=(0,1,3), names=None, converters=conv)
-        control = np.rec.array([[1,5,0], [2,8,1], [3,3,3]], dtype=dtyp)
+        control = np.rec.array([(1,5,0), (2,8,1), (3,3,3)], dtype=dtyp)
         assert_equal(test, control)
 
     def test_dtype_with_object(self):
diff --git a/numpy/ma/tests/test_core.py b/numpy/ma/tests/test_core.py
--- a/numpy/ma/tests/test_core.py
+++ b/numpy/ma/tests/test_core.py
@@ -1760,15 +1760,11 @@ def test_check_on_fields(self):
         assert_equal(fval.item(), [-999, -12345678.9, b"???"])
 
         #.....Using a flexible type w/ a different type shouldn't matter
-        # BEHAVIOR in 1.5 and earlier: match structured types by position
-        #fill_val = np.array((-999, -12345678.9, "???"),
-        #                    dtype=[("A", int), ("B", float), ("C", "|S3")])
-        # BEHAVIOR in 1.6 and later: match structured types by name
-        fill_val = np.array(("???", -999, -12345678.9),
-                            dtype=[("c", "|S3"), ("a", int), ("b", float), ])
-        # suppress deprecation warning in 1.12 (remove in 1.13)
-        with assert_warns(FutureWarning):
-            fval = _check_fill_value(fill_val, ndtype)
+        # BEHAVIOR in 1.5 and earlier, and 1.13 and later: match structured
+        # types by position
+        fill_val = np.array((-999, -12345678.9, "???"),
+                            dtype=[("A", int), ("B", float), ("C", "|S3")])
+        fval = _check_fill_value(fill_val, ndtype)
         assert_(isinstance(fval, ndarray))
         assert_equal(fval.item(), [-999, -12345678.9, b"???"])