Repository: numpy/numpy

Issue:
simple improvement to make testing utils work with ndarray subclasses (Trac #1021)
_Original ticket http://projects.scipy.org/numpy/ticket/1021 on 2009-02-22 by @ddale, assigned to unknown._

I found a simple way to modify numpy's testing utils that makes it easier to use them for unittesting ndarray subclasses. I tried running numpy's (svn 6456) test suite both with and without the patch applied, and I got identical results: 1 known failure and 11 skips on a 64-bit linux platform.

Its a really small patch, any chance of getting it in before 1.3 is released?



Discussion:

Resolving patch:
diff --git a/numpy/add_newdocs.py b/numpy/add_newdocs.py
--- a/numpy/add_newdocs.py
+++ b/numpy/add_newdocs.py
@@ -5563,7 +5563,8 @@ def luf(lamdaexpr, *args, **kwargs):
     align : bool, optional
         Add padding to the fields to match what a C compiler would output
         for a similar C-struct. Can be ``True`` only if `obj` is a dictionary
-        or a comma-separated string.
+        or a comma-separated string. If a struct dtype is being created,
+        this also sets a sticky alignment flag ``isalignedstruct``.
     copy : bool, optional
         Make a new copy of the data-type object. If ``False``, the result
         may just be a reference to a built-in data-type object.
@@ -5787,6 +5788,14 @@ def luf(lamdaexpr, *args, **kwargs):
 
     """))
 
+add_newdoc('numpy.core.multiarray', 'dtype', ('isalignedstruct',
+    """
+    Boolean indicating whether the dtype is a struct which maintains
+    field alignment. This flag is sticky, so when combining multiple
+    structs together, it is preserved and produces new dtypes which
+    are also aligned.
+    """))
+
 add_newdoc('numpy.core.multiarray', 'dtype', ('itemsize',
     """
     The element size of this data-type object.
diff --git a/numpy/core/_internal.py b/numpy/core/_internal.py
--- a/numpy/core/_internal.py
+++ b/numpy/core/_internal.py
@@ -12,7 +12,7 @@
 else:
     _nbo = asbytes('>')
 
-def _makenames_list(adict):
+def _makenames_list(adict, align):
     from multiarray import dtype
     allfields = []
     fnames = adict.keys()
@@ -26,7 +26,7 @@ def _makenames_list(adict):
         num = int(obj[1])
         if (num < 0):
             raise ValueError("invalid offset.")
-        format = dtype(obj[0])
+        format = dtype(obj[0], align=align)
         if (format.itemsize == 0):
             raise ValueError("all itemsizes must be fixed.")
         if (n > 2):
@@ -53,7 +53,7 @@ def _usefields(adict, align):
     except KeyError:
         names = None
     if names is None:
-        names, formats, offsets, titles = _makenames_list(adict)
+        names, formats, offsets, titles = _makenames_list(adict, align)
     else:
         formats = []
         offsets = []


Test patch:
diff --git a/numpy/core/tests/test_dtype.py b/numpy/core/tests/test_dtype.py
--- a/numpy/core/tests/test_dtype.py
+++ b/numpy/core/tests/test_dtype.py
@@ -41,6 +41,25 @@ def test_invalid_types(self):
             #print typestr
             assert_raises(TypeError, np.dtype, typestr)
 
+    def test_bad_param(self):
+        # Can't give a size that's too small
+        assert_raises(ValueError, np.dtype,
+                        {'names':['f0','f1'],
+                         'formats':['i4', 'i1'],
+                         'offsets':[0,4],
+                         'itemsize':4})
+        # If alignment is enabled, the alignment (4) must divide the itemsize
+        assert_raises(ValueError, np.dtype,
+                        {'names':['f0','f1'],
+                         'formats':['i4', 'i1'],
+                         'offsets':[0,4],
+                         'itemsize':9}, align=True)
+        # If alignment is enabled, the individual fields must be aligned
+        assert_raises(ValueError, np.dtype,
+                        {'names':['f0','f1'],
+                         'formats':['i1','f4'],
+                         'offsets':[0,2]}, align=True)
+
 class TestRecord(TestCase):
     def test_equivalent_record(self):
         """Test whether equivalent record dtypes hash the same."""
@@ -56,10 +75,12 @@ def test_different_names(self):
 
     def test_different_titles(self):
         # In theory, they may hash the same (collision) ?
-        a = np.dtype({'names': ['r','b'], 'formats': ['u1', 'u1'],
-            'titles': ['Red pixel', 'Blue pixel']})
-        b = np.dtype({'names': ['r','b'], 'formats': ['u1', 'u1'],
-            'titles': ['RRed pixel', 'Blue pixel']})
+        a = np.dtype({'names': ['r','b'],
+                      'formats': ['u1', 'u1'],
+                      'titles': ['Red pixel', 'Blue pixel']})
+        b = np.dtype({'names': ['r','b'],
+                      'formats': ['u1', 'u1'],
+                      'titles': ['RRed pixel', 'Blue pixel']})
         assert_dtype_not_equal(a, b)
 
     def test_not_lists(self):
@@ -71,6 +92,99 @@ def test_not_lists(self):
         self.assertRaises(TypeError, np.dtype,
             dict(names=['A', 'B'], formats=set(['f8', 'i4'])))
 
+    def test_aligned_size(self):
+        # Check that structured dtypes get padded to an aligned size
+        dt = np.dtype('i4, i1', align=True)
+        assert_equal(dt.itemsize, 8)
+        dt = np.dtype([('f0', 'i4'), ('f1', 'i1')], align=True)
+        assert_equal(dt.itemsize, 8)
+        dt = np.dtype({'names':['f0','f1'],
+                       'formats':['i4', 'u1'],
+                       'offsets':[0,4]}, align=True)
+        assert_equal(dt.itemsize, 8)
+        dt = np.dtype({'f0': ('i4', 0), 'f1':('u1', 4)}, align=True)
+        assert_equal(dt.itemsize, 8)
+        # Nesting should preserve that alignment
+        dt1 = np.dtype([('f0', 'i4'),
+                       ('f1', [('f1', 'i1'), ('f2', 'i4'), ('f3', 'i1')]),
+                       ('f2', 'i1')], align=True)
+        assert_equal(dt1.itemsize, 20)
+        dt2 = np.dtype({'names':['f0','f1','f2'],
+                       'formats':['i4',
+                                  [('f1', 'i1'), ('f2', 'i4'), ('f3', 'i1')],
+                                  'i1'],
+                       'offsets':[0, 4, 16]}, align=True)
+        assert_equal(dt2.itemsize, 20)
+        dt3 = np.dtype({'f0': ('i4', 0),
+                       'f1': ([('f1', 'i1'), ('f2', 'i4'), ('f3', 'i1')], 4),
+                       'f2': ('i1', 16)}, align=True)
+        assert_equal(dt3.itemsize, 20)
+        assert_equal(dt1, dt2)
+        assert_equal(dt2, dt3)
+        # Nesting should preserve packing
+        dt1 = np.dtype([('f0', 'i4'),
+                       ('f1', [('f1', 'i1'), ('f2', 'i4'), ('f3', 'i1')]),
+                       ('f2', 'i1')], align=False)
+        assert_equal(dt1.itemsize, 11)
+        dt2 = np.dtype({'names':['f0','f1','f2'],
+                       'formats':['i4',
+                                  [('f1', 'i1'), ('f2', 'i4'), ('f3', 'i1')],
+                                  'i1'],
+                       'offsets':[0, 4, 10]}, align=False)
+        assert_equal(dt2.itemsize, 11)
+        dt3 = np.dtype({'f0': ('i4', 0),
+                       'f1': ([('f1', 'i1'), ('f2', 'i4'), ('f3', 'i1')], 4),
+                       'f2': ('i1', 10)}, align=False)
+        assert_equal(dt3.itemsize, 11)
+        assert_equal(dt1, dt2)
+        assert_equal(dt2, dt3)
+
+    def test_union_struct(self):
+        # Should be able to create union dtypes
+        dt = np.dtype({'names':['f0','f1','f2'], 'formats':['<u4', '<u2','<u2'],
+                        'offsets':[0,0,2]}, align=True)
+        assert_equal(dt.itemsize, 4)
+        a = np.array([3], dtype='<u4').view(dt)
+        a['f1'] = 10
+        a['f2'] = 36
+        assert_equal(a['f0'], 10 + 36*256*256)
+        # Should be able to specify fields out of order
+        dt = np.dtype({'names':['f0','f1','f2'], 'formats':['<u4', '<u2','<u2'],
+                        'offsets':[4,0,2]}, align=True)
+        assert_equal(dt.itemsize, 8)
+        dt2 = np.dtype({'names':['f2','f0','f1'],
+                        'formats':['<u2', '<u4','<u2'],
+                        'offsets':[2,4,0]}, align=True)
+        vals = [(0,1,2), (3,-1,4)]
+        vals2 = [(2,0,1), (4,3,-1)]
+        a = np.array(vals, dt)
+        b = np.array(vals2, dt2)
+        assert_equal(a.astype(dt2), b)
+        assert_equal(b.astype(dt), a)
+        assert_equal(a.view(dt2), b)
+        assert_equal(b.view(dt), a)
+        # Should not be able to overlap objects with other types
+        assert_raises(TypeError, np.dtype,
+                {'names':['f0','f1'],
+                 'formats':['O', 'i1'],
+                 'offsets':[0,2]})
+        assert_raises(TypeError, np.dtype,
+                {'names':['f0','f1'],
+                 'formats':['i4', 'O'],
+                 'offsets':[0,3]})
+        assert_raises(TypeError, np.dtype,
+                {'names':['f0','f1'],
+                 'formats':[[('a','O')], 'i1'],
+                 'offsets':[0,2]})
+        assert_raises(TypeError, np.dtype,
+                {'names':['f0','f1'],
+                 'formats':['i4', [('a','O')]],
+                 'offsets':[0,3]})
+        # Out of order should still be ok, however
+        dt = np.dtype({'names':['f0','f1'],
+                       'formats':['i1', 'O'],
+                       'offsets':[np.dtype('intp').itemsize, 0]})
+
 class TestSubarray(TestCase):
     def test_single_subarray(self):
         a = np.dtype((np.int, (2)))
@@ -165,5 +279,65 @@ def test_nested_metadata(self):
         d = np.dtype([('a', np.dtype(int, metadata={'datum': 1}))])
         self.assertEqual(d['a'].metadata, {'datum': 1})
 
+class TestString(TestCase):
+    def test_complex_dtype_str(self):
+        dt = np.dtype([('top', [('tiles', ('>f4', (64, 64)), (1,)),
+                                ('rtile', '>f4', (64, 36))], (3,)),
+                       ('bottom', [('bleft', ('>f4', (8, 64)), (1,)),
+                                   ('bright', '>f4', (8, 36))])])
+        assert_equal(str(dt),
+                     "[('top', [('tiles', ('>f4', (64, 64)), (1,)), "
+                     "('rtile', '>f4', (64, 36))], (3,)), "
+                     "('bottom', [('bleft', ('>f4', (8, 64)), (1,)), "
+                     "('bright', '>f4', (8, 36))])]")
+
+        dt = np.dtype({'names': ['r','g','b'], 'formats': ['u1', 'u1', 'u1'],
+                        'offsets': [0, 1, 2],
+                        'titles': ['Red pixel', 'Green pixel', 'Blue pixel']})
+        assert_equal(str(dt),
+                    "[(('Red pixel', 'r'), 'u1'), "
+                    "(('Green pixel', 'g'), 'u1'), "
+                    "(('Blue pixel', 'b'), 'u1')]")
+
+        dt = np.dtype({'names': ['r','b'], 'formats': ['u1', 'u1'],
+                        'offsets': [0, 2],
+                        'titles': ['Red pixel', 'Blue pixel']})
+        assert_equal(str(dt),
+                    "{'names':['r','b'], "
+                    "'formats':['u1','u1'], "
+                    "'offsets':[0,2], "
+                    "'titles':['Red pixel','Blue pixel']}")
+
+    def test_complex_dtype_repr(self):
+        dt = np.dtype([('top', [('tiles', ('>f4', (64, 64)), (1,)),
+                                ('rtile', '>f4', (64, 36))], (3,)),
+                       ('bottom', [('bleft', ('>f4', (8, 64)), (1,)),
+                                   ('bright', '>f4', (8, 36))])])
+        assert_equal(repr(dt),
+                     "dtype([('top', [('tiles', ('>f4', (64, 64)), (1,)), "
+                     "('rtile', '>f4', (64, 36))], (3,)), "
+                     "('bottom', [('bleft', ('>f4', (8, 64)), (1,)), "
+                     "('bright', '>f4', (8, 36))])])")
+
+        dt = np.dtype({'names': ['r','g','b'], 'formats': ['u1', 'u1', 'u1'],
+                        'offsets': [0, 1, 2],
+                        'titles': ['Red pixel', 'Green pixel', 'Blue pixel']},
+                        align=True)
+        assert_equal(repr(dt),
+                    "dtype([(('Red pixel', 'r'), 'u1'), "
+                    "(('Green pixel', 'g'), 'u1'), "
+                    "(('Blue pixel', 'b'), 'u1')], align=True)")
+
+        dt = np.dtype({'names': ['r','b'], 'formats': ['u1', 'u1'],
+                        'offsets': [0, 2],
+                        'titles': ['Red pixel', 'Blue pixel'],
+                        'itemsize': 4})
+        assert_equal(repr(dt),
+                    "dtype({'names':['r','b'], "
+                    "'formats':['u1','u1'], "
+                    "'offsets':[0,2], "
+                    "'titles':['Red pixel','Blue pixel'], "
+                    "'itemsize':4})")
+
 if __name__ == "__main__":
     run_module_suite()
diff --git a/numpy/core/tests/test_regression.py b/numpy/core/tests/test_regression.py
--- a/numpy/core/tests/test_regression.py
+++ b/numpy/core/tests/test_regression.py
@@ -853,17 +853,6 @@ def test_floats_from_string(self, level=rlevel):
         assert_almost_equal(fdouble, 1.234)
         assert_almost_equal(flongdouble, 1.234)
 
-    def test_complex_dtype_printing(self, level=rlevel):
-        dt = np.dtype([('top', [('tiles', ('>f4', (64, 64)), (1,)),
-                                ('rtile', '>f4', (64, 36))], (3,)),
-                       ('bottom', [('bleft', ('>f4', (8, 64)), (1,)),
-                                   ('bright', '>f4', (8, 36))])])
-        assert_equal(str(dt),
-                     "[('top', [('tiles', ('>f4', (64, 64)), (1,)), "
-                     "('rtile', '>f4', (64, 36))], (3,)), "
-                     "('bottom', [('bleft', ('>f4', (8, 64)), (1,)), "
-                     "('bright', '>f4', (8, 36))])]")
-
     def test_nonnative_endian_fill(self, level=rlevel):
         """ Non-native endian arrays were incorrectly filled with scalars before
         r5034.