Repository: lebedov/msgpack-numpy

Issue:
Failures unpacking nested, structured dtypes.
`msgpack_numpy` fails when unpacking nested, structured dtypes. 

See https://numpy.org/doc/stable/user/basics.rec.html for description.

Example:

```python
structured_dtype = np.dtype([("a", float), ("b", int)])
nested_dtype = np.dtype([("foo", structured_dtype), ("bar", structured_dtype)])

x = np.empty((10,), dtype=nested_dtype)
x["foo"]["a"] = np.arange(10)
x["foo"]["b"] = np.arange(10)
x["bar"]["a"] = np.arange(10) + 10
x["bar"]["b"] = np.arange(10) + 10

unpackb(packb(x))
```

Fails with:

```
  File "/home/alexford/asford/msgpack_numpy/msgpack-numpy/msgpack_numpy.py", line 260, in unpackb
    return _unpackb(packed, **kwargs)
  File "msgpack/_unpacker.pyx", line 195, in msgpack._cmsgpack.unpackb
  File "/home/alexford/asford/msgpack_numpy/msgpack-numpy/msgpack_numpy.py", line 91, in decode
    dtype=np.dtype(descr)).reshape(obj[b'shape'])
TypeError: Field elements must be 2- or 3-tuples, got '['a', '<f8']'
```

Comments:

Patch:
diff --git a/msgpack_numpy.py b/msgpack_numpy.py
index 48e6df5..38d89a2 100644
--- a/msgpack_numpy.py
+++ b/msgpack_numpy.py
@@ -88,11 +88,11 @@ def decode(obj, chain=None):
                 else:
                     descr = obj[b'type']
                 return np.frombuffer(obj[b'data'],
-                            dtype=np.dtype(descr)).reshape(obj[b'shape'])
+                            dtype=_unpack_dtype(descr)).reshape(obj[b'shape'])
             else:
                 descr = obj[b'type']
                 return np.frombuffer(obj[b'data'],
-                            dtype=np.dtype(descr))[0]
+                            dtype=_unpack_dtype(descr))[0]
         elif b'complex' in obj:
             return complex(tostr(obj[b'data']))
         else:
@@ -100,6 +100,18 @@ def decode(obj, chain=None):
     except KeyError:
         return obj if chain is None else chain(obj)
 
+def _unpack_dtype(dtype):
+    """
+    Unpack dtype descr, recursively unpacking nested structured dtypes.
+    """
+
+    if isinstance(dtype, (list, tuple)):
+        dtype = [
+            (name, _unpack_dtype(subdtype)) + tuple(rest)
+            for name, subdtype, *rest in dtype
+        ]
+    return np.dtype(dtype)
+
 if msgpack.version < (1, 0, 0):
     warnings.warn('support for msgpack < 1.0.0 will be removed in a future release',
                    DeprecationWarning)


Test patch:
diff --git a/tests.py b/tests.py
index cb0ad55..f84876b 100644
--- a/tests.py
+++ b/tests.py
@@ -243,5 +243,44 @@ class test_numpy_msgpack(TestCase):
         x_rec = self.encode_decode_thirdparty(x)
         self.assertEqual(x, x_rec)
 
+    def test_numpy_structured_array(self):
+        structured_dtype = np.dtype([("a", float), ("b", int)])
+
+        x = np.empty((10,), dtype=structured_dtype)
+        x["a"] = np.arange(10)
+        x["b"] = np.arange(10)
+
+        x_rec = self.encode_decode(x)
+
+        assert_array_equal(x, x_rec)
+        self.assertEqual(x.dtype, x_rec.dtype)
+
+    def test_numpy_shaped_structured_array(self):
+        shaped_structured_dtype = np.dtype([("a", float, 3), ("b", int)])
+
+        x = np.empty((10,), dtype=shaped_structured_dtype)
+        x["a"] = np.arange(30).reshape(10, 3)
+        x["b"] = np.arange(10)
+
+        x_rec = self.encode_decode(x)
+
+        assert_array_equal(x, x_rec)
+        self.assertEqual(x.dtype, x_rec.dtype)
+
+    def test_numpy_nested_structured_array(self):
+        structured_dtype = np.dtype([("a", float), ("b", int)])
+        nested_dtype = np.dtype([("foo", structured_dtype), ("bar", structured_dtype)])
+
+        x = np.empty((10,), dtype=nested_dtype)
+        x["foo"]["a"] = np.arange(10)
+        x["foo"]["b"] = np.arange(10)
+        x["bar"]["a"] = np.arange(10) + 10
+        x["bar"]["b"] = np.arange(10) + 10
+
+        x_rec = self.encode_decode(x)
+
+        assert_array_equal(x, x_rec)
+        self.assertEqual(x.dtype, x_rec.dtype)
+
 if __name__ == '__main__':
     main()