Repository: numba/numba

Issue:
Consolidate array methods testing
Test for various array methods are scatter across two files: : test_array_methods and test_array_manipulation.  We should consolidate them.



Discussion:
Hmm, the point of splitting them was to make the test modules more readable (and also shorter to run when run individually).

Clarify: consolidate testing of the same array method into one place.  We are testing reshape, for example, in both file.

Ah, right, sorry for the misunderstanding.


Resolving patch:
diff --git a/numba/targets/arrayobj.py b/numba/targets/arrayobj.py
--- a/numba/targets/arrayobj.py
+++ b/numba/targets/arrayobj.py
@@ -2679,6 +2679,36 @@ def array_copy(context, builder, sig, args):
     return impl_ret_new_ref(context, builder, sig.return_type, ret._getvalue())
 
 
+@lower_builtin("array.astype", types.Array, types.DTypeSpec)
+def array_astype(context, builder, sig, args):
+    arytype = sig.args[0]
+    ary = make_array(arytype)(context, builder, value=args[0])
+    shapes = cgutils.unpack_tuple(builder, ary.shape)
+
+    rettype = sig.return_type
+    ret = _empty_nd_impl(context, builder, rettype, shapes)
+
+    src_data = ary.data
+    dest_data = ret.data
+
+    src_strides = cgutils.unpack_tuple(builder, ary.strides)
+    dest_strides = cgutils.unpack_tuple(builder, ret.strides)
+    intp_t = context.get_value_type(types.intp)
+
+    with cgutils.loop_nest(builder, shapes, intp_t) as indices:
+        src_ptr = cgutils.get_item_pointer2(builder, src_data,
+                                            shapes, src_strides,
+                                            arytype.layout, indices)
+        dest_ptr = cgutils.get_item_pointer2(builder, dest_data,
+                                             shapes, dest_strides,
+                                             rettype.layout, indices)
+        item = load_item(context, builder, arytype, src_ptr)
+        item = context.cast(builder, item, arytype.dtype, rettype.dtype)
+        store_item(context, builder, rettype, item, dest_ptr)
+
+    return impl_ret_new_ref(context, builder, sig.return_type, ret._getvalue())
+
+
 @lower_builtin(numpy.frombuffer, types.Buffer)
 @lower_builtin(numpy.frombuffer, types.Buffer, types.DTypeSpec)
 def np_frombuffer(context, builder, sig, args):
diff --git a/numba/typing/arraydecl.py b/numba/typing/arraydecl.py
--- a/numba/typing/arraydecl.py
+++ b/numba/typing/arraydecl.py
@@ -337,9 +337,27 @@ def resolve_view(self, ary, args, kws):
         assert not kws
         dtype, = args
         dtype = _parse_dtype(dtype)
+        if dtype is None:
+            return
         retty = ary.copy(dtype=dtype)
         return signature(retty, *args)
 
+    @bound_function("array.astype")
+    def resolve_astype(self, ary, args, kws):
+        from .npydecl import _parse_dtype
+        assert not kws
+        dtype, = args
+        dtype = _parse_dtype(dtype)
+        if dtype is None:
+            return
+        if not self.context.can_convert(ary.dtype, dtype):
+            raise TypeError("astype(%s) not supported on %s: "
+                            "cannot convert from %s to %s"
+                            % (dtype, ary, ary.dtype, dtype))
+        layout = ary.layout if ary.layout in 'CF' else 'C'
+        retty = ary.copy(dtype=dtype, layout=layout)
+        return signature(retty, *args)
+
     @bound_function("array.ravel")
     def resolve_ravel(self, ary, args, kws):
         # Only support no argument version (default order='C')


Test patch:
diff --git a/numba/tests/test_array_manipulation.py b/numba/tests/test_array_manipulation.py
--- a/numba/tests/test_array_manipulation.py
+++ b/numba/tests/test_array_manipulation.py
@@ -1,11 +1,13 @@
 from __future__ import print_function
-import numpy as np
+
 from functools import partial
+import numba.unittest_support as unittest
+
+import numpy as np
 
 from numba.compiler import compile_isolated, Flags
 from numba import types, from_dtype, errors, typeof
-import numba.unittest_support as unittest
-from numba.tests.support import TestCase, MemoryLeakMixin
+from .support import TestCase, MemoryLeakMixin, CompilationCache, tag
 
 enable_pyobj_flags = Flags()
 enable_pyobj_flags.set("enable_pyobject")
@@ -14,12 +16,8 @@
 no_pyobj_flags.set('nrt')
 
 
-def reshape_array(a):
-    return a.reshape(3, 3)
-
-
-def reshape_array_to_1d(a):
-    return a.reshape(a.size)
+def array_reshape(arr, newshape):
+    return arr.reshape(newshape)
 
 
 def flatten_array(a):
@@ -46,16 +44,6 @@ def squeeze_array(a):
     return a.squeeze()
 
 
-def convert_array_str(a):
-    # astype takes no kws argument in numpy1.6
-    return a.astype('f4')
-
-
-def convert_array_dtype(a):
-    # astype takes no kws argument in numpy1.6
-    return a.astype(np.float32)
-
-
 def add_axis1(a):
     return np.expand_dims(a, axis=0)
 
@@ -78,72 +66,87 @@ def bad_float_index(arr):
 
 
 class TestArrayManipulation(MemoryLeakMixin, TestCase):
-    def test_reshape_array(self, flags=enable_pyobj_flags):
-        a = np.arange(9)
-
-        pyfunc = reshape_array
-        arraytype1 = typeof(a)
-        cr = compile_isolated(pyfunc, (arraytype1,), flags=flags)
-        cfunc = cr.entry_point
-
-        expected = pyfunc(a)
-        got = cfunc(a)
-        np.testing.assert_equal(expected, got)
-
-    def test_reshape_array_npm(self):
-        self.test_reshape_array(flags=no_pyobj_flags)
-
-    def test_reshape_array_to_1d(self, flags=enable_pyobj_flags,
-                                 layout='C'):
-        a = np.arange(9).reshape(3, 3)
-        if layout == 'F':
-            a = a.T
-
-        pyfunc = reshape_array_to_1d
-        arraytype1 = typeof(a)
-        if layout == 'A':
-            # Force A layout
-            arraytype1 = arraytype1.copy(layout='A')
-
-        self.assertEqual(arraytype1.layout, layout)
-        cr = compile_isolated(pyfunc, (arraytype1,), flags=flags)
-        cfunc = cr.entry_point
-
-        expected = pyfunc(a)
-        got = cfunc(a)
-        self.assertEqual(got.ndim, 1)
-        np.testing.assert_equal(expected, got)
-
-    def test_reshape_array_to_1d_npm(self):
-        self.test_reshape_array_to_1d(flags=no_pyobj_flags)
-        with self.assertRaises(NotImplementedError) as raises:
-            self.test_reshape_array_to_1d(flags=no_pyobj_flags, layout='F')
-        self.assertIn("incompatible shape for array", str(raises.exception))
-        with self.assertTypingError() as raises:
-            # The following will leak due to lack of post exception cleanup
-            self.test_reshape_array_to_1d(flags=no_pyobj_flags, layout='A')
-        self.assertIn("reshape() supports contiguous array only",
-                      str(raises.exception))
-        # Disable leak check for the last `test_reshape_array_to_1d` call.
+    """
+    Check shape-changing operations on arrays.
+    """
+
+    def setUp(self):
+        super(TestArrayManipulation, self).setUp()
+        self.ccache = CompilationCache()
+
+    @tag('important')
+    def test_array_reshape(self):
+        pyfunc = array_reshape
+        def run(arr, shape):
+            cres = self.ccache.compile(pyfunc, (typeof(arr), typeof(shape)))
+            return cres.entry_point(arr, shape)
+        def check(arr, shape):
+            expected = pyfunc(arr, shape)
+            self.memory_leak_setup()
+            got = run(arr, shape)
+            self.assertPreciseEqual(got, expected)
+            del got
+            self.memory_leak_teardown()
+        def check_err_shape(arr, shape):
+            with self.assertRaises(NotImplementedError) as raises:
+                run(arr, shape)
+            self.assertEqual(str(raises.exception),
+                             "incompatible shape for array")
+        def check_err_size(arr, shape):
+            with self.assertRaises(ValueError) as raises:
+                run(arr, shape)
+            self.assertEqual(str(raises.exception),
+                             "total size of new array must be unchanged")
+
+        def check_err_multiple_negative(arr, shape):
+            with self.assertRaises(ValueError) as raises:
+                run(arr, shape)
+            self.assertEqual(str(raises.exception),
+                             "multiple negative shape value")
+
+        # C-contiguous
+        arr = np.arange(24)
+        check(arr, (24,))
+        check(arr, (4, 6))
+        check(arr, (8, 3))
+        check(arr, (8, 1, 3))
+        check(arr, (1, 8, 1, 1, 3, 1))
+        arr = np.arange(24).reshape((2, 3, 4))
+        check(arr, (24,))
+        check(arr, (4, 6))
+        check(arr, (8, 3))
+        check(arr, (8, 1, 3))
+        check(arr, (1, 8, 1, 1, 3, 1))
+        check_err_size(arr, (25,))
+        check_err_size(arr, (8, 4))
+        arr = np.arange(24).reshape((1, 8, 1, 1, 3, 1))
+        check(arr, (24,))
+        check(arr, (4, 6))
+        check(arr, (8, 3))
+        check(arr, (8, 1, 3))
+
+        # F-contiguous
+        arr = np.arange(24).reshape((2, 3, 4)).T
+        check(arr, (4, 3, 2))
+        check(arr, (1, 4, 1, 3, 1, 2, 1))
+        check_err_shape(arr, (2, 3, 4))
+        check_err_shape(arr, (6, 4))
+        check_err_shape(arr, (2, 12))
+
+        # Test negative shape value
+        arr = np.arange(25).reshape(5,5)
+        check(arr, -1)
+        check(arr, (-1,))
+        check(arr, (-1, 5))
+        check(arr, (5, -1, 5))
+        check(arr, (5, 5, -1))
+        check_err_size(arr, (-1, 4))
+        check_err_multiple_negative(arr, (-1, -2, 5, 5))
+        check_err_multiple_negative(arr, (5, 5, -1, -1))
+
+        # Exceptions leak references
         self.disable_leak_check()
 
-    @unittest.expectedFailure
-    def test_reshape_array_to_1d_leak_error_npm(self):
-        """
-        Rerun the test in ``test_reshape_array_to_1d_npm`` that will cause
-        a leak error.
-        """
-        with self.assertRaises(NotImplementedError) as raises:
-            self.test_reshape_array_to_1d(flags=no_pyobj_flags, layout='F')
-        self.assertIn("incompatible shape for array", str(raises.exception))
-        # The leak check is not captured by the expectedFailure.
-        # We need to disable it because `test_reshape_array_to_1d` will leak
-        # due to the lack of post exception cleanup
-        self.disable_leak_check()
-        # The following checks for memory leak and it will fail.
-        # This will trigger the expectedFailure
-        self.assert_no_memory_leak()
-
     def test_flatten_array(self, flags=enable_pyobj_flags, layout='C'):
         a = np.arange(9).reshape(3, 3)
         if layout == 'F':
@@ -254,42 +257,6 @@ def test_squeeze_array_npm(self):
 
         self.assertIn("squeeze", str(raises.exception))
 
-    def test_convert_array_str(self, flags=enable_pyobj_flags):
-        a = np.arange(9, dtype='i4')
-
-        pyfunc = convert_array_str
-        arraytype1 = typeof(a)
-        cr = compile_isolated(pyfunc, (arraytype1,), flags=flags)
-        cfunc = cr.entry_point
-
-        expected = pyfunc(a)
-        got = cfunc(a)
-        self.assertPreciseEqual(expected, got)
-
-    def test_convert_array_str_npm(self):
-        with self.assertRaises(errors.UntypedAttributeError) as raises:
-            self.test_convert_array_str(flags=no_pyobj_flags)
-
-        self.assertIn("astype", str(raises.exception))
-
-    def test_convert_array(self, flags=enable_pyobj_flags):
-        a = np.arange(9, dtype='i4')
-
-        pyfunc = convert_array_dtype
-        arraytype1 = typeof(a)
-        cr = compile_isolated(pyfunc, (arraytype1,), flags=flags)
-        cfunc = cr.entry_point
-
-        expected = pyfunc(a)
-        got = cfunc(a)
-        np.testing.assert_equal(expected, got)
-
-    def test_convert_array_npm(self):
-        with self.assertRaises(errors.UntypedAttributeError) as raises:
-            self.test_convert_array(flags=no_pyobj_flags)
-
-        self.assertIn("astype", str(raises.exception))
-
     def test_add_axis1(self, flags=enable_pyobj_flags):
         a = np.arange(9).reshape(3, 3)
 
diff --git a/numba/tests/test_array_methods.py b/numba/tests/test_array_methods.py
--- a/numba/tests/test_array_methods.py
+++ b/numba/tests/test_array_methods.py
@@ -65,9 +65,6 @@ def array_transpose(arr):
 def array_copy(arr):
     return arr.copy()
 
-def array_reshape(arr, newshape):
-    return arr.reshape(newshape)
-
 def array_view(arr, newtype):
     return arr.view(newtype)
 
@@ -80,6 +77,12 @@ def array_view(arr):
 def array_sliced_view(arr, ):
     return arr[0:4].view(np.float32)[0]
 
+def make_array_astype(newtype):
+    def array_astype(arr):
+        return arr.astype(newtype)
+    return array_astype
+
+
 def np_frombuffer(b):
     """
     np.frombuffer() on a Python-allocated buffer.
@@ -116,15 +119,51 @@ def np_where_3(c, x, y):
     return np.where(c, x, y)
 
 
-class TestArrayMethodsCustom(MemoryLeak, TestCase):
+class TestArrayMethods(MemoryLeakMixin, TestCase):
     """
-    Test np.round, np.around, ndarray.reshape
+    Test various array methods and array-related functions.
     """
 
     def setUp(self):
-        super(TestArrayMethodsCustom, self).setUp()
+        super(TestArrayMethods, self).setUp()
         self.ccache = CompilationCache()
 
+    def check_round_scalar(self, unary_pyfunc, binary_pyfunc):
+        base_values = [-3.0, -2.5, -2.25, -1.5, 1.5, 2.25, 2.5, 2.75]
+        complex_values = [x * (1 - 1j) for x in base_values]
+        int_values = [int(x) for x in base_values]
+        argtypes = (types.float64, types.float32, types.int32,
+                    types.complex64, types.complex128)
+        argvalues = [base_values, base_values, int_values,
+                     complex_values, complex_values]
+
+        pyfunc = binary_pyfunc
+        for ty, values in zip(argtypes, argvalues):
+            cres = compile_isolated(pyfunc, (ty, types.int32))
+            cfunc = cres.entry_point
+            for decimals in (1, 0, -1):
+                for v in values:
+                    if decimals > 0:
+                        v *= 10
+                    expected = _fixed_np_round(v, decimals)
+                    got = cfunc(v, decimals)
+                    self.assertPreciseEqual(got, expected)
+
+        pyfunc = unary_pyfunc
+        for ty, values in zip(argtypes, argvalues):
+            cres = compile_isolated(pyfunc, (ty,))
+            cfunc = cres.entry_point
+            for v in values:
+                expected = _fixed_np_round(v)
+                got = cfunc(v)
+                self.assertPreciseEqual(got, expected)
+
+    def test_round_scalar(self):
+        self.check_round_scalar(np_round_unary, np_round_binary)
+
+    def test_around_scalar(self):
+        self.check_round_scalar(np_around_unary, np_around_binary)
+
     def check_round_array(self, pyfunc):
         def check_round(cfunc, values, inty, outty, decimals):
             # Create input and output arrays of the right type
@@ -169,82 +208,15 @@ def check_types(argtypes, outtypes, values):
         argtypes = (types.complex64, types.complex128)
         check_types(argtypes, argtypes, values * (1 - 1j))
 
+        # Exceptions leak references
+        self.disable_leak_check()
+
     def test_round_array(self):
         self.check_round_array(np_round_array)
 
     def test_around_array(self):
         self.check_round_array(np_around_array)
 
-    @tag('important')
-    def test_array_reshape(self):
-        pyfunc = array_reshape
-        def run(arr, shape):
-            cres = self.ccache.compile(pyfunc, (typeof(arr), typeof(shape)))
-            return cres.entry_point(arr, shape)
-        def check(arr, shape):
-            expected = pyfunc(arr, shape)
-            self.memory_leak_setup()
-            got = run(arr, shape)
-            self.assertPreciseEqual(got, expected)
-            del got
-            self.memory_leak_teardown()
-        def check_err_shape(arr, shape):
-            with self.assertRaises(NotImplementedError) as raises:
-                run(arr, shape)
-            self.assertEqual(str(raises.exception),
-                             "incompatible shape for array")
-        def check_err_size(arr, shape):
-            with self.assertRaises(ValueError) as raises:
-                run(arr, shape)
-            self.assertEqual(str(raises.exception),
-                             "total size of new array must be unchanged")
-
-        def check_err_multiple_negative(arr, shape):
-            with self.assertRaises(ValueError) as raises:
-                run(arr, shape)
-            self.assertEqual(str(raises.exception),
-                             "multiple negative shape value")
-
-        # C-contiguous
-        arr = np.arange(24)
-        check(arr, (24,))
-        check(arr, (4, 6))
-        check(arr, (8, 3))
-        check(arr, (8, 1, 3))
-        check(arr, (1, 8, 1, 1, 3, 1))
-        arr = np.arange(24).reshape((2, 3, 4))
-        check(arr, (24,))
-        check(arr, (4, 6))
-        check(arr, (8, 3))
-        check(arr, (8, 1, 3))
-        check(arr, (1, 8, 1, 1, 3, 1))
-        check_err_size(arr, (25,))
-        check_err_size(arr, (8, 4))
-        arr = np.arange(24).reshape((1, 8, 1, 1, 3, 1))
-        check(arr, (24,))
-        check(arr, (4, 6))
-        check(arr, (8, 3))
-        check(arr, (8, 1, 3))
-
-        # F-contiguous
-        arr = np.arange(24).reshape((2, 3, 4)).T
-        check(arr, (4, 3, 2))
-        check(arr, (1, 4, 1, 3, 1, 2, 1))
-        check_err_shape(arr, (2, 3, 4))
-        check_err_shape(arr, (6, 4))
-        check_err_shape(arr, (2, 12))
-
-        # Test negative shape value
-        arr = np.arange(25).reshape(5,5)
-        check(arr, -1)
-        check(arr, (-1,))
-        check(arr, (-1, 5))
-        check(arr, (5, -1, 5))
-        check(arr, (5, 5, -1))
-        check_err_size(arr, (-1, 4))
-        check_err_multiple_negative(arr, (-1, -2, 5, 5))
-        check_err_multiple_negative(arr, (5, 5, -1, -1))
-
     def test_array_view(self):
 
         def run(arr, dtype):
@@ -319,6 +291,9 @@ def check_err(arr, dtype):
         check_err(arr, dt1)
         check_err(arr, dt2)
 
+        # Exceptions leak references
+        self.disable_leak_check()
+
     def test_array_sliced_view(self):
         """
         Test .view() on A layout array but has contiguous innermost dimension.
@@ -335,6 +310,32 @@ def test_array_sliced_view(self):
 
         self.assertEqual(expect, got)
 
+    def test_array_astype(self):
+
+        def run(arr, dtype):
+            pyfunc = make_array_astype(dtype)
+            cres = self.ccache.compile(pyfunc, (typeof(arr),))
+            return cres.entry_point(arr)
+        def check(arr, dtype):
+            expected = arr.astype(dtype).copy(order='A')
+            got = run(arr, dtype)
+            self.assertPreciseEqual(got, expected)
+
+        # C-contiguous
+        arr = np.arange(24, dtype=np.int8)
+        check(arr, np.dtype('int16'))
+        check(arr, np.int32)
+        check(arr, np.float32)
+        check(arr, np.complex128)
+
+        # F-contiguous
+        arr = np.arange(24, dtype=np.int8).reshape((3, 8)).T
+        check(arr, np.float32)
+
+        # Non-contiguous
+        arr = np.arange(16, dtype=np.int32)[::2]
+        check(arr, np.uint64)
+
     def check_np_frombuffer(self, pyfunc):
         def run(buf):
             cres = self.ccache.compile(pyfunc, (typeof(buf),))
@@ -360,6 +361,9 @@ def check(buf):
         b = np.arange(12).reshape((3, 4))
         check(b)
 
+        # Exceptions leak references
+        self.disable_leak_check()
+
         with self.assertRaises(ValueError) as raises:
             run(bytearray(b"xxx"))
         self.assertEqual("buffer size must be a multiple of element size",
@@ -371,52 +375,6 @@ def test_np_frombuffer(self):
     def test_np_frombuffer_dtype(self):
         self.check_np_frombuffer(np_frombuffer_dtype)
 
-
-class TestArrayMethods(MemoryLeakMixin, TestCase):
-    """
-    Test various array methods and array-related functions.
-    """
-
-    def setUp(self):
-        super(TestArrayMethods, self).setUp()
-        self.ccache = CompilationCache()
-
-    def check_round_scalar(self, unary_pyfunc, binary_pyfunc):
-        base_values = [-3.0, -2.5, -2.25, -1.5, 1.5, 2.25, 2.5, 2.75]
-        complex_values = [x * (1 - 1j) for x in base_values]
-        int_values = [int(x) for x in base_values]
-        argtypes = (types.float64, types.float32, types.int32,
-                    types.complex64, types.complex128)
-        argvalues = [base_values, base_values, int_values,
-                     complex_values, complex_values]
-
-        pyfunc = binary_pyfunc
-        for ty, values in zip(argtypes, argvalues):
-            cres = compile_isolated(pyfunc, (ty, types.int32))
-            cfunc = cres.entry_point
-            for decimals in (1, 0, -1):
-                for v in values:
-                    if decimals > 0:
-                        v *= 10
-                    expected = _fixed_np_round(v, decimals)
-                    got = cfunc(v, decimals)
-                    self.assertPreciseEqual(got, expected)
-
-        pyfunc = unary_pyfunc
-        for ty, values in zip(argtypes, argvalues):
-            cres = compile_isolated(pyfunc, (ty,))
-            cfunc = cres.entry_point
-            for v in values:
-                expected = _fixed_np_round(v)
-                got = cfunc(v)
-                self.assertPreciseEqual(got, expected)
-
-    def test_round_scalar(self):
-        self.check_round_scalar(np_round_unary, np_round_binary)
-
-    def test_around_scalar(self):
-        self.check_round_scalar(np_around_unary, np_around_binary)
-
     def check_layout_dependent_func(self, pyfunc, fac=np.arange):
         def check_arr(arr):
             cres = compile_isolated(pyfunc, (typeof(arr),))