Repository: numpy/numpy

Issue:
Cryptic SystemError when creating array with weird structured-but-empty dtype
Trying to create an array with a weird structured but empty dtype results in a `SystemError`:

```
In [214]: array([(), (), (), (), ()], dtype={'names':[], 'formats':[], 'offsets':[], 'itemsize':12})
---------------------------------------------------------------------------
SystemError                               Traceback (most recent call last)
<ipython-input-214-1b509554769d> in <module>()
----> 1 array([(), (), (), (), ()], dtype={'names':[], 'formats':[], 'offsets':[], 'itemsize':12})

SystemError: error return without exception set
```

This should be caught somewhere and a suitable exception should be raised instead of `SystemError`.

Cryptic SystemError when creating array with weird structured-but-empty dtype
Trying to create an array with a weird structured but empty dtype results in a `SystemError`:

```
In [214]: array([(), (), (), (), ()], dtype={'names':[], 'formats':[], 'offsets':[], 'itemsize':12})
---------------------------------------------------------------------------
SystemError                               Traceback (most recent call last)
<ipython-input-214-1b509554769d> in <module>()
----> 1 array([(), (), (), (), ()], dtype={'names':[], 'formats':[], 'offsets':[], 'itemsize':12})

SystemError: error return without exception set
```

This should be caught somewhere and a suitable exception should be raised instead of `SystemError`.



Discussion:

Resolving patch:
diff --git a/numpy/core/_internal.py b/numpy/core/_internal.py
--- a/numpy/core/_internal.py
+++ b/numpy/core/_internal.py
@@ -10,7 +10,10 @@
 import sys
 import warnings
 
-from numpy.compat import asbytes, bytes
+from numpy.compat import asbytes, bytes, basestring
+from .multiarray import dtype, array, ndarray
+import ctypes
+from .numerictypes import object_
 
 if (sys.byteorder == 'little'):
     _nbo = asbytes('<')
@@ -18,7 +21,6 @@
     _nbo = asbytes('>')
 
 def _makenames_list(adict, align):
-    from .multiarray import dtype
     allfields = []
     fnames = list(adict.keys())
     for fname in fnames:
@@ -52,7 +54,6 @@ def _makenames_list(adict, align):
 #  a dictionary without "names" and "formats"
 #  fields is used as a data-type descriptor.
 def _usefields(adict, align):
-    from .multiarray import dtype
     try:
         names = adict[-1]
     except KeyError:
@@ -130,7 +131,6 @@ def _array_descr(descriptor):
 # so don't remove the name here, or you'll
 # break backward compatibilty.
 def _reconstruct(subtype, shape, dtype):
-    from .multiarray import ndarray
     return ndarray.__new__(subtype, shape, dtype)
 
 
@@ -194,12 +194,10 @@ def _commastring(astr):
     return result
 
 def _getintp_ctype():
-    from .multiarray import dtype
     val = _getintp_ctype.cache
     if val is not None:
         return val
     char = dtype('p').char
-    import ctypes
     if (char == 'i'):
         val = ctypes.c_int
     elif char == 'l':
@@ -224,7 +222,6 @@ def c_void_p(self, num):
 class _ctypes(object):
     def __init__(self, array, ptr=None):
         try:
-            import ctypes
             self._ctypes = ctypes
         except ImportError:
             self._ctypes = _missing_ctypes()
@@ -287,23 +284,55 @@ def _newnames(datatype, order):
         return tuple(list(order) + nameslist)
     raise ValueError("unsupported order value: %s" % (order,))
 
-# Given an array with fields and a sequence of field names
-# construct a new array with just those fields copied over
-def _index_fields(ary, fields):
-    from .multiarray import empty, dtype, array
+def _index_fields(ary, names):
+    """ Given a structured array and a sequence of field names
+    construct new array with just those fields.
+
+    Parameters
+    ----------
+    ary : ndarray
+        Structured array being subscripted
+    names : string or list of strings
+        Either a single field name, or a list of field names
+
+    Returns
+    -------
+    sub_ary : ndarray
+        If `names` is a single field name, the return value is identical to
+        ary.getfield, a writeable view into `ary`. If `names` is a list of
+        field names the return value is a copy of `ary` containing only those
+        fields. This is planned to return a view in the future.
+    
+    Raises
+    ------
+    ValueError
+        If `ary` does not contain a field given in `names`.
+
+    """
     dt = ary.dtype
 
-    names = [name for name in fields if name in dt.names]
-    formats = [dt.fields[name][0] for name in fields if name in dt.names]
-    offsets = [dt.fields[name][1] for name in fields if name in dt.names]
+    #use getfield to index a single field
+    if isinstance(names, basestring):
+        try:
+            return ary.getfield(dt.fields[names][0], dt.fields[names][1])
+        except KeyError:
+            raise ValueError("no field of name %s" % names)
+
+    for name in names:
+        if name not in dt.fields:
+            raise ValueError("no field of name %s" % name)
 
-    view_dtype = {'names':names, 'formats':formats, 'offsets':offsets, 'itemsize':dt.itemsize}
-    view = ary.view(dtype=view_dtype)
+    formats = [dt.fields[name][0] for name in names]
+    offsets = [dt.fields[name][1] for name in names]
+
+    view_dtype = {'names': names, 'formats': formats,
+                  'offsets': offsets, 'itemsize': dt.itemsize}
+
+    # return copy for now (future plan to return ary.view(dtype=view_dtype))
+    copy_dtype = {'names': view_dtype['names'],
+                  'formats': view_dtype['formats']}
+    return array(ary.view(dtype=view_dtype), dtype=copy_dtype, copy=True)
 
-    # Return a copy for now until behavior is fully deprecated
-    # in favor of returning view
-    copy_dtype = {'names':view_dtype['names'], 'formats':view_dtype['formats']}
-    return array(view, dtype=copy_dtype, copy=True)
 
 def _get_all_field_offsets(dtype, base_offset=0):
     """ Returns the types and offsets of all fields in a (possibly structured)
@@ -363,8 +392,6 @@ def _check_field_overlap(new_fields, old_fields):
         If the new fields are incompatible with the old fields
 
     """
-    from .numerictypes import object_
-    from .multiarray import dtype
 
     #first go byte by byte and check we do not access bytes not in old_fields
     new_bytes = set()
@@ -527,8 +554,6 @@ def _view_is_safe(oldtype, newtype):
 _pep3118_standard_typechars = ''.join(_pep3118_standard_map.keys())
 
 def _dtype_from_pep3118(spec, byteorder='@', is_subdtype=False):
-    from numpy.core.multiarray import dtype
-
     fields = {}
     offset = 0
     explicit_name = False
@@ -694,8 +719,6 @@ def get_dummy_name():
 
 def _add_trailing_padding(value, padding):
     """Inject the specified number of padding bytes at the end of a dtype"""
-    from numpy.core.multiarray import dtype
-
     if value.fields is None:
         vfields = {'f0': (value, 0)}
     else:


Test patch:
diff --git a/numpy/core/tests/test_indexing.py b/numpy/core/tests/test_indexing.py
--- a/numpy/core/tests/test_indexing.py
+++ b/numpy/core/tests/test_indexing.py
@@ -409,6 +409,10 @@ def __getitem__(self, item):
         arr = np.arange(10)
         assert_array_equal(arr[SequenceLike()], arr[SequenceLike(),])
 
+        # also test that field indexing does not segfault
+        # for a similar reason, by indexing a structured array
+        arr = np.zeros((1,), dtype=[('f1', 'i8'), ('f2', 'i8')])
+        assert_array_equal(arr[SequenceLike()], arr[SequenceLike(),])
 
 class TestFieldIndexing(TestCase):
     def test_scalar_return_type(self):
diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py
--- a/numpy/core/tests/test_multiarray.py
+++ b/numpy/core/tests/test_multiarray.py
@@ -3484,7 +3484,7 @@ def test_bytes_fields(self):
             assert_raises(ValueError, dt.__getitem__, asbytes('a'))
 
             x = np.array([(1,), (2,), (3,)], dtype=dt)
-            assert_raises(ValueError, x.__getitem__, asbytes('a'))
+            assert_raises(IndexError, x.__getitem__, asbytes('a'))
 
             y = x[0]
             assert_raises(IndexError, y.__getitem__, asbytes('a'))
@@ -3517,8 +3517,8 @@ def test_field_names(self):
         if is_py3:
             funcs = (str,)
             # byte string indexing fails gracefully
-            assert_raises(ValueError, a.__setitem__, asbytes('f1'), 1)
-            assert_raises(ValueError, a.__getitem__, asbytes('f1'))
+            assert_raises(IndexError, a.__setitem__, asbytes('f1'), 1)
+            assert_raises(IndexError, a.__getitem__, asbytes('f1'))
             assert_raises(IndexError, a['f1'].__setitem__, asbytes('sf1'), 1)
             assert_raises(IndexError, a['f1'].__getitem__, asbytes('sf1'))
         else:
@@ -3564,7 +3564,7 @@ def test_field_names(self):
 
     def test_field_names_deprecation(self):
 
-        def collect_warning_types(f, *args, **kwargs):
+        def collect_warnings(f, *args, **kwargs):
             with warnings.catch_warnings(record=True) as log:
                 warnings.simplefilter("always")
                 f(*args, **kwargs)
@@ -3585,20 +3585,19 @@ def collect_warning_types(f, *args, **kwargs):
 
         # All the different functions raise a warning, but not an error, and
         # 'a' is not modified:
-        assert_equal(collect_warning_types(a[['f1', 'f2']].__setitem__, 0, (10, 20)),
+        assert_equal(collect_warnings(a[['f1', 'f2']].__setitem__, 0, (10, 20)),
                      [FutureWarning])
         assert_equal(a, b)
         # Views also warn
         subset = a[['f1', 'f2']]
         subset_view = subset.view()
-        assert_equal(collect_warning_types(subset_view['f1'].__setitem__, 0, 10),
+        assert_equal(collect_warnings(subset_view['f1'].__setitem__, 0, 10),
                      [FutureWarning])
         # But the write goes through:
         assert_equal(subset['f1'][0], 10)
-        # Only one warning per multiple field indexing, though (even if there are
-        # multiple views involved):
-        assert_equal(collect_warning_types(subset['f1'].__setitem__, 0, 10),
-                     [])
+        # Only one warning per multiple field indexing, though (even if there
+        # are multiple views involved):
+        assert_equal(collect_warnings(subset['f1'].__setitem__, 0, 10), [])
 
     def test_record_hash(self):
         a = np.array([(1, 2), (1, 2)], dtype='i1,i2')
@@ -3616,11 +3615,18 @@ def test_record_no_hash(self):
         a = np.array([(1, 2), (1, 2)], dtype='i1,i2')
         self.assertRaises(TypeError, hash, a[0])
 
+    def test_empty_structure_creation(self):
+        # make sure these do not raise errors (gh-5631)
+        array([()], dtype={'names': [], 'formats': [],
+                           'offsets': [], 'itemsize': 12})
+        array([(), (), (), (), ()], dtype={'names': [], 'formats': [],
+                                           'offsets': [], 'itemsize': 12})
+
 class TestView(TestCase):
     def test_basic(self):
         x = np.array([(1, 2, 3, 4), (5, 6, 7, 8)],
-                      dtype=[('r', np.int8), ('g', np.int8),
-                             ('b', np.int8), ('a', np.int8)])
+                     dtype=[('r', np.int8), ('g', np.int8),
+                            ('b', np.int8), ('a', np.int8)])
         # We must be specific about the endianness here:
         y = x.view(dtype='<i4')
         # ... and again without the keyword.
diff --git a/numpy/core/tests/test_records.py b/numpy/core/tests/test_records.py
--- a/numpy/core/tests/test_records.py
+++ b/numpy/core/tests/test_records.py
@@ -235,6 +235,11 @@ def test_record_scalar_setitem(self):
         rec[0].x = 1
         assert_equal(rec[0].x, np.ones(5))
 
+    def test_missing_field(self):
+        # https://github.com/numpy/numpy/issues/4806
+        arr = np.zeros((3,), dtype=[('x', int), ('y', int)])
+        assert_raises(ValueError, lambda: arr[['nofield']])
+
 def test_find_duplicate():
     l1 = [1, 2, 3, 4, 5, 6]
     assert_(np.rec.find_duplicate(l1) == [])