Repository: numpy/numpy

Issue:
np.load() "invalid shape in fixed-type tuple" in NumPy 1.16.0
In NumPy 1.16.0, `np.load()` fails to load some files created by `np.save()`, saying "ValueError: invalid shape in fixed-type tuple."

The problem seems to be that #12358 changed `np.lib.format._read_array_header()`.  It used to pass the descr to `np.dtype()`, but now it uses a new function `descr_to_dtype()` which fails for some valid inputs.

We have significant amounts of data stored on disk with formats which NumPy 1.16.0 cannot load, so this is a showstopper for upgrading beyond 1.15.

### Reproducing code example:
This fails in NumPy 1.16.0, but works in all prior versions:
```python
import numpy as np
a = np.zeros(1, np.dtype([('c', ('<f8', (5,)), (2,))]))
np.save('a.npy', a)
np.load('a.npy')
```

### Error message:
```
Traceback (most recent call last):
  File "numpy/lib/npyio.py", line 440, in load
    pickle_kwargs=pickle_kwargs)
  File "numpy/lib/format.py", line 678, in read_array
    shape, fortran_order, dtype = _read_array_header(fp, version)
  File "numpy/lib/format.py", line 563, in _read_array_header
    dtype = descr_to_dtype(d['descr'])
  File "numpy/lib/format.py", line 282, in descr_to_dtype
    dt = numpy.dtype((descr_to_dtype(descr_str), shape))
  File "numpy/lib/format.py", line 282, in descr_to_dtype
    dt = numpy.dtype((descr_to_dtype(descr_str), shape))
ValueError: invalid shape in fixed-type tuple.
```

### Numpy/Python version information:
NumPy 1.16.0
Python 3.6.6


Discussion:

Resolving patch:
diff --git a/numpy/lib/format.py b/numpy/lib/format.py
--- a/numpy/lib/format.py
+++ b/numpy/lib/format.py
@@ -261,15 +261,19 @@ def dtype_to_descr(dtype):
 def descr_to_dtype(descr):
     '''
     descr may be stored as dtype.descr, which is a list of
-    (name, format, [shape]) tuples. Offsets are not explicitly saved, rather
-    empty fields with name,format == '', '|Vn' are added as padding.
+    (name, format, [shape]) tuples where format may be a str or a tuple.
+    Offsets are not explicitly saved, rather empty fields with
+    name, format == '', '|Vn' are added as padding.
 
     This function reverses the process, eliminating the empty padding fields.
     '''
-    if isinstance(descr, (str, dict)):
+    if isinstance(descr, str):
         # No padding removal needed
         return numpy.dtype(descr)
-
+    elif isinstance(descr, tuple):
+        # subtype, will always have a shape descr[1]
+        dt = descr_to_dtype(descr[0])
+        return numpy.dtype((dt, descr[1]))
     fields = []
     offset = 0
     for field in descr:


Test patch:
diff --git a/numpy/lib/tests/test_format.py b/numpy/lib/tests/test_format.py
--- a/numpy/lib/tests/test_format.py
+++ b/numpy/lib/tests/test_format.py
@@ -411,6 +411,7 @@ def teardown_module():
     np.array(NbufferT, dtype=np.dtype(Ndescr).newbyteorder('<')),
     np.array(PbufferT, dtype=np.dtype(Pdescr).newbyteorder('>')),
     np.array(NbufferT, dtype=np.dtype(Ndescr).newbyteorder('>')),
+    np.zeros(1, dtype=[('c', ('<f8', (5,)), (2,))])
 ]
 
 
@@ -628,6 +629,61 @@ def test_pickle_disallow():
     assert_raises(ValueError, np.save, path, np.array([None], dtype=object),
                   allow_pickle=False)
 
+@pytest.mark.parametrize('dt', [
+    np.dtype(np.dtype([('a', np.int8),
+                       ('b', np.int16),
+                       ('c', np.int32),
+                      ], align=True),
+             (3,)),
+    np.dtype([('x', np.dtype({'names':['a','b'],
+                              'formats':['i1','i1'],
+                              'offsets':[0,4],
+                              'itemsize':8,
+                             },
+                    (3,)),
+               (4,),
+             )]),
+    np.dtype([('x',
+                   ('<f8', (5,)),
+                   (2,),
+               )]),
+    np.dtype([('x', np.dtype((
+        np.dtype((
+            np.dtype({'names':['a','b'],
+                      'formats':['i1','i1'],
+                      'offsets':[0,4],
+                      'itemsize':8}),
+            (3,)
+            )),
+        (4,)
+        )))
+        ]),
+    np.dtype([
+        ('a', np.dtype((
+            np.dtype((
+                np.dtype((
+                    np.dtype([
+                        ('a', int),
+                        ('b', np.dtype({'names':['a','b'],
+                                        'formats':['i1','i1'],
+                                        'offsets':[0,4],
+                                        'itemsize':8})),
+                    ]),
+                    (3,),
+                )),
+                (4,),
+            )),
+            (5,),
+        )))
+        ]),
+    ])
+
+def test_descr_to_dtype(dt):
+    dt1 = format.descr_to_dtype(dt.descr)
+    assert_equal_(dt1, dt)
+    arr1 = np.zeros(3, dt)
+    arr2 = roundtrip(arr1)
+    assert_array_equal(arr1, arr2)
 
 def test_version_2_0():
     f = BytesIO()