Repository: numba/numba

Issue:
Structured dtypes with multi-dim fields: numba either throws or crashes
Given this (quite trivial) example:

```python
dtype1 = np.dtype([('a', 'i8'), ('b', 'i4')])
dtype2 = np.dtype((dtype1, (2, 2)))
dtype3 = np.dtype([('x', '?'), ('y', dtype2)])
arr = np.zeros(2, dtype3)

@numba.jit(nopython=True)
def foo(arr):
    return arr[0]

foo(arr)
```

numba will most likely throw like so:

```
.../numba/types/npytypes.py in __init__(self, dtype, shape)
    419 
    420     def __init__(self, dtype, shape):
--> 421         assert dtype.bitwidth % 8 == 0, \
    422             "Dtype bitwidth must be a multiple of bytes"
    423         self._shape = shape

AttributeError: 'Record' object has no attribute 'bitwidth'
```

... or you could also get

```
SystemError: 
CPUDispatcher(<function foo at 0x7f7741382d08>) returned NULL without setting an error
```


Discussion:
Thanks for the report. I can reproduce the raised `AttributeError`, no luck with reproducing the `SystemError` as yet. This is clearly a bug.
This bug occurs in much simpler cases:
```
Item = np.dtype([('some_field', np.int32)])
SomeList = np.dtype([('items', Item, 10)])
```

Accessing `items` property of an instance of `SomeList` crashes with this message:
`'Record' object has no attribute 'bitwidth'`

A quick fix is to replace `NestedArray.size`'s body inside `npytypes.py` with something like this:
```
        if isinstance(self.dtype, Record):
            return self.dtype.size
        else:
            return self.dtype.bitwidth // 8
```

P.S. I would increase this bug's priority - it seems like a fairly common situation.
@kazhoyan thanks, this is an exact reproducer:
```python
from numba import njit
import numpy as np

Item = np.dtype([('some_field', np.int32)])
SomeList = np.dtype([('items', Item, 10)])

arr = np.zeros((2,), SomeList)

@njit
def foo(x):
    x[0]

foo.py_func(arr)
foo(arr)
```
Pull Requests to fix/promote discussion around such issues are welcomed.
This still open? 
Any solution till now?
> This still open?

Yes, still reproduces on 0.54.

> Any solution till now?

Not that I'm aware of I'm afraid.

Possible starter patch for this and #1469:

```diff
diff --git a/numba/core/types/npytypes.py b/numba/core/types/npytypes.py
index 25050f30c..e9da0249d 100644
--- a/numba/core/types/npytypes.py
+++ b/numba/core/types/npytypes.py
@@ -209,6 +209,10 @@ class Record(Type):
 
         return as_struct_dtype(self)
 
+    @property
+    def bitwidth(self):
+        return self.dtype.itemsize * 8
+
     def can_convert_to(self, typingctx, other):
         """
         Convert this Record to the *other*.
@@ -427,6 +431,10 @@ class Array(Buffer):
         if (not aligned or
             (isinstance(dtype, Record) and not dtype.aligned)):
             self.aligned = False
+        if isinstance(dtype, NestedArray):
+            tmp = Array(dtype.dtype, dtype.ndim, 'C')
+            ndim += tmp.ndim
+            dtype = tmp.dtype
         if name is None:
             type_name = "array"
             if not self.mutable:
@@ -553,6 +561,10 @@ class NestedArray(Array):
     """
 
     def __init__(self, dtype, shape):
+        if isinstance(dtype, NestedArray):
+            tmp = Array(dtype.dtype, dtype.ndim, 'C')
+            shape += dtype.shape
+            dtype = tmp.dtype
         assert dtype.bitwidth % 8 == 0, \
             "Dtype bitwidth must be a multiple of bytes"
         self._shape = shape
```

This works around the issue, but the approach might need to be more cleanly thought-out (and tested).

With a modified version of @stuartarchibald's reproducer above (to print out return values):

```python
from numba import njit
import numpy as np

Item = np.dtype([('some_field', np.int32)])
SomeList = np.dtype([('items', Item, 10)])

arr = np.zeros((2,), SomeList)

@njit
def foo(x):
    return x[0]

print(foo.py_func(arr))
print(foo(arr))
```

I get

```
([(0,), (0,), (0,), (0,), (0,), (0,), (0,), (0,), (0,), (0,)],)
([(0,), (0,), (0,), (0,), (0,), (0,), (0,), (0,), (0,), (0,)],)
```

with this patch.

Resolving patch:
diff --git a/numba/core/types/npytypes.py b/numba/core/types/npytypes.py
--- a/numba/core/types/npytypes.py
+++ b/numba/core/types/npytypes.py
@@ -134,6 +134,8 @@ def __init__(self, fields, size, aligned):
         name = 'Record({};{};{})'.format(desc, self.size, self.aligned)
         super(Record, self).__init__(name)
 
+        self.bitwidth = self.dtype.itemsize * 8
+
     @classmethod
     def _normalize_fields(cls, fields):
         """
@@ -430,6 +432,9 @@ def __init__(self, dtype, ndim, layout, readonly=False, name=None,
         if (not aligned or
             (isinstance(dtype, Record) and not dtype.aligned)):
             self.aligned = False
+        if isinstance(dtype, NestedArray):
+            ndim += dtype.ndim
+            dtype = dtype.dtype
         if name is None:
             type_name = "array"
             if not self.mutable:
@@ -556,6 +561,10 @@ class NestedArray(Array):
     """
 
     def __init__(self, dtype, shape):
+        if isinstance(dtype, NestedArray):
+            tmp = Array(dtype.dtype, dtype.ndim, 'C')
+            shape += dtype.shape
+            dtype = tmp.dtype
         assert dtype.bitwidth % 8 == 0, \
             "Dtype bitwidth must be a multiple of bytes"
         self._shape = shape


Test patch:
diff --git a/numba/tests/test_record_dtype.py b/numba/tests/test_record_dtype.py
--- a/numba/tests/test_record_dtype.py
+++ b/numba/tests/test_record_dtype.py
@@ -1732,6 +1732,73 @@ def copy(index, src, dest):
 
         np.testing.assert_array_equal(expected, got)
 
+    def test_issue_1469_1(self):
+        # Dimensions of nested arrays as a dtype are concatenated with the
+        # dimensions of the array.
+        nptype = np.dtype((np.float64, (4,)))
+        nbtype = types.Array(numpy_support.from_dtype(nptype), 2, 'C')
+        expected = types.Array(types.float64, 3, 'C')
+        self.assertEqual(nbtype, expected)
+
+    def test_issue_1469_2(self):
+        # Nesting nested arrays results in concatentated dimensions -
+        # In this example a 3D array of a 2D nested array of 2D nested arrays
+        # results in a 7D type.
+        nptype = np.dtype((np.dtype((np.float64, (5, 2))), (3, 6)))
+        nbtype = types.Array(numpy_support.from_dtype(nptype), 3, 'C')
+        expected = types.Array(types.float64, 7, 'C')
+        self.assertEqual(nbtype, expected)
+
+    def test_issue_1469_3(self):
+        # Nested arrays in record dtypes are accepted, but alignment is not
+        # guaranteed.
+        nptype = np.dtype([('a', np.float64,(4,))])
+        nbtype = types.Array(numpy_support.from_dtype(nptype), 2, 'C')
+
+        # Manual construction of expected array of record type
+        natype = types.NestedArray(types.float64, (4,))
+        fields = [('a', {'type': natype, 'offset': 0})]
+        rectype = types.Record(fields=fields, size=32, aligned=False)
+        expected = types.Array(rectype, 2, 'C', aligned=False)
+
+        self.assertEqual(nbtype, expected)
+
+    def test_issue_3158_1(self):
+        # A nested array dtype.
+        item = np.dtype([('some_field', np.int32)])
+        items = np.dtype([('items', item, 3)])
+
+        @njit
+        def fn(x):
+            return x[0]
+
+        arr = np.asarray([([(0,), (1,), (2,)],),
+                          ([(3,), (4,), (5,)],)],
+                         dtype=items)
+
+        expected = fn.py_func(arr)
+        actual = fn(arr)
+
+        self.assertEqual(expected, actual)
+
+    def test_issue_3158_2(self):
+        # A slightly more complex nested array dtype example.
+        dtype1 = np.dtype([('a', 'i8'), ('b', 'i4')])
+        dtype2 = np.dtype((dtype1, (2, 2)))
+        dtype3 = np.dtype([('x', '?'), ('y', dtype2)])
+
+        @njit
+        def fn(arr):
+            return arr[0]
+
+        arr = np.asarray([(False, [[(0, 1), (2, 3)], [(4, 5), (6, 7)]]),
+                          (True, [[(8, 9), (10, 11)], [(12, 13), (14, 15)]])],
+                         dtype=dtype3)
+        expected = fn.py_func(arr)
+        actual = fn(arr)
+
+        self.assertEqual(expected, actual)
+
 
 if __name__ == '__main__':
     unittest.main()