Repository: numba/numba

Issue:
Regression for structured arrays with titles

- [X] I am using the latest released version of Numba (most recent is visible in
 the change log (https://github.com/numba/numba/blob/master/CHANGE_LOG).
- [X] I have included below a minimal working reproducer (if you are unsure how
 to write one see http://matthewrocklin.com/blog/work/2018/02/28/minimal-bug-reports).

Since v0.43, numba has problems with structured arrays with titles. Example:
```python
import numba
import numpy as np

x = np.zeros(2, dtype=[
    (("title a", 'a'), np.float),
    ('b', np.float)])

@numba.jit
def f(x):
    return x

print("Before: ", x.dtype)
print("After:  ", f(x).dtype)
```

In numba 0.42 this gives the expected:
```
Before:  [(('title a', 'a'), '<f8'), ('b', '<f8')]
After:   [(('title a', 'a'), '<f8'), ('b', '<f8')]
```

But in 0.43 and the latest master, it gives:
```
Before:  [(('title a', 'a'), '<f8'), ('b', '<f8')]
After:   {'names':['a','title a','b'], 'formats':['<f8','<f8','<f8'], 'offsets':[0,0,8], 'itemsize':16}
```
The title is gone from field a, and instead a new field has been added to the array. This new field has the same offset, so the array now has overlapping fields. This will crash things like compressors that view the array as a memory buffer, with an error like:
```
ValueError: dtypes with overlapping or out-of-order fields are not representable as buffers. Consider reordering the fields.
```

I'm guessing this new behaviour was introduced with the Record refactor in #3661?

I attempted a naive fix by replacing the iteration over `dtype.fields` here: https://github.com/numba/numba/blob/e9518295f1c5461137d62c9735d525896f2c37fb/numba/numpy_support.py#L420 with 
```python
    for name in dtype.names:
        info = dtype.fields[name]
```
which removes the extra fields. Unfortunately, this would break accessing a field by its title, and doesn't resolve the fact that titles of fields are gone after passing through numba.



Discussion:
Thanks for the report. Indeed this was altered in #3661. Ping @sklam.

Resolving patch:
diff --git a/numba/numpy_support.py b/numba/numpy_support.py
--- a/numba/numpy_support.py
+++ b/numba/numpy_support.py
@@ -154,12 +154,24 @@ def as_struct_dtype(rec):
     """Convert Numba Record type to NumPy structured dtype
     """
     assert isinstance(rec, types.Record)
-    names = [k for k, _ in rec.members]
+    names = []
+    formats = []
+    offsets = []
+    titles = []
+    # Fill the fields if they are not a title.
+    for k, t in rec.members:
+        if not rec.is_title(k):
+            names.append(k)
+            formats.append(as_dtype(t))
+            offsets.append(rec.offset(k))
+            titles.append(rec.fields[k].title)
+
     fields = {
         'names': names,
-        'formats': [as_dtype(t) for _, t in rec.members],
-        'offsets': [rec.offset(k) for k in names],
+        'formats': formats,
+        'offsets': offsets,
         'itemsize': rec.size,
+        'titles': titles,
     }
     _check_struct_alignment(rec, fields)
     return np.dtype(fields, align=rec.aligned)
@@ -434,12 +446,15 @@ def from_struct_dtype(dtype):
 
     fields = []
     for name, info in dtype.fields.items():
-        # *info* may have 3 element if it has a "title", which can be ignored
+        # *info* may have 3 element
         [elemdtype, offset] = info[:2]
+        title = info[2] if len(info) == 3 else None
+
         ty = from_dtype(elemdtype)
         infos = {
             'type': ty,
             'offset': offset,
+            'title': title,
         }
         fields.append((name, infos))
 
diff --git a/numba/types/npytypes.py b/numba/types/npytypes.py
--- a/numba/types/npytypes.py
+++ b/numba/types/npytypes.py
@@ -44,7 +44,10 @@ def key(self):
         return self.count
 
 
-_RecordField = collections.namedtuple('_RecordField', 'type,offset,alignment')
+_RecordField = collections.namedtuple(
+    '_RecordField',
+    'type,offset,alignment,title',
+)
 
 
 class Record(Type):
@@ -103,10 +106,11 @@ def __init__(self, fields, size, aligned):
         descbuf = []
         fmt = "{}[type={};offset={}{}]"
         for k, infos in fields:
+            extra = ""
             if infos.alignment is not None:
-                extra = ';alignment={}'.format(infos.alignment)
-            else:
-                extra = ''
+                extra += ';alignment={}'.format(infos.alignment)
+            elif infos.title is not None:
+                extra += ';title={}'.format(infos.title)
             descbuf.append(fmt.format(k, infos.type, infos.offset, extra))
 
         desc = ','.join(descbuf)
@@ -117,7 +121,13 @@ def __init__(self, fields, size, aligned):
     def _normalize_fields(cls, fields):
         """
         fields:
-            [name: str, { type: Type, offset: int, [ alignment: int ] }]
+            [name: str,
+             value: {
+                 type: Type,
+                 offset: int,
+                 [ alignment: int ],
+                 [ title : str],
+             }]
         """
         res = []
         for name, infos in sorted(fields, key=lambda x: (x[1]['offset'], x[0])):
@@ -125,6 +135,7 @@ def _normalize_fields(cls, fields):
                 type=infos['type'],
                 offset=infos['offset'],
                 alignment=infos.get('alignment'),
+                title=infos.get('title'),
             )
             res.append((name, fd))
         return res
@@ -161,6 +172,16 @@ def alignof(self, key):
         """
         return self.fields[key].alignment
 
+    def has_titles(self):
+        """Returns True the record uses titles.
+        """
+        return any(fd.title is not None for fd in self.fields.values())
+
+    def is_title(self, key):
+        """Returns True if the field named *key* is a title.
+        """
+        return self.fields[key].title == key
+
     @property
     def members(self):
         """An ordered list of (name, type) for the fields.


Test patch:
diff --git a/numba/tests/test_numpy_support.py b/numba/tests/test_numpy_support.py
--- a/numba/tests/test_numpy_support.py
+++ b/numba/tests/test_numpy_support.py
@@ -121,20 +121,20 @@ def check(dtype, fields, size, aligned):
 
         dtype = np.dtype([('a', np.int16), ('b', np.int32)])
         check(dtype,
-              fields={'a': (types.int16, 0, None),
-                      'b': (types.int32, 2, None)},
+              fields={'a': (types.int16, 0, None, None),
+                      'b': (types.int32, 2, None, None)},
               size=6, aligned=False)
 
         dtype = np.dtype([('a', np.int16), ('b', np.int32)], align=True)
         check(dtype,
-              fields={'a': (types.int16, 0, None),
-                      'b': (types.int32, 4, None)},
+              fields={'a': (types.int16, 0, None, None),
+                      'b': (types.int32, 4, None, None)},
               size=8, aligned=True)
 
         dtype = np.dtype([('m', np.int32), ('n', 'S5')])
         check(dtype,
-              fields={'m': (types.int32, 0, None),
-                      'n': (types.CharSeq(5), 4, None)},
+              fields={'m': (types.int32, 0, None, None),
+                      'n': (types.CharSeq(5), 4, None, None)},
               size=9, aligned=False)
 
     @tag('important')
diff --git a/numba/tests/test_record_dtype.py b/numba/tests/test_record_dtype.py
--- a/numba/tests/test_record_dtype.py
+++ b/numba/tests/test_record_dtype.py
@@ -812,6 +812,15 @@ def pyfunc(a):
         got = cfunc(arr.copy())
         np.testing.assert_equal(expect, got)
 
+    def test_record_dtype_with_titles_roundtrip(self):
+        recdtype = np.dtype([(("title a", 'a'), np.float), ('b', np.float)])
+        nbtype = numpy_support.from_dtype(recdtype)
+        self.assertTrue(nbtype.is_title('title a'))
+        self.assertFalse(nbtype.is_title('a'))
+        self.assertFalse(nbtype.is_title('b'))
+        got = numpy_support.as_dtype(nbtype)
+        self.assertTrue(got, recdtype)
+
 
 def _get_cfunc_nopython(pyfunc, argspec):
     return jit(argspec, nopython=True)(pyfunc)