Repository: cupy/cupy
Pull request: Optimize the initialization of `List[ndarray]` in `cupy.array`
- Implement `cupy.array(<List[cupy.ndarray]>)`
- Improve the performance of `cupy.array(<List[numpy.ndarray]>)`

Fixes #409 and #1084. Ref #1030.

Problem statement:
cupy.asarray doesn't accept list of cupy.ndarray
Both `numpy.asarray` and `cupy.asarray` accept list of `numpy.ndarray`:

```
>>> numpy.asarray([numpy.zeros(1)])
array([[ 0.]])
>>> cupy.asarray([numpy.zeros(1)])
array([[ 0.]])
```

However, `cupy.asarray` doesn't accept list of `cupy.ndarray`:

```
>>> cupy.asarray([cupy.zeros(1)])
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/Users/user/.pyenv/versions/flatten-obs/lib/python3.6/site-packages/cupy/creation/from_data.py", line 46, in asarray
    return core.array(a, dtype, False)
  File "cupy/core/core.pyx", line 1799, in cupy.core.core.array (cupy/core/core.cpp:58790)
  File "cupy/core/core.pyx", line 1824, in cupy.core.core.array (cupy/core/core.cpp:58526)
ValueError: Unsupported dtype object
```

This behavior might be consistent in some sense, but I often find it cumbersome when writing numpy/cupy agnostic code.

Hints from issue discussion:
When writing code with Chainer, `Unsupported dtype object` was one of the common issue, especially in pre-processing code that is both compatible with NumPy/CuPy.
It is hard to guess the reason from this error message for first-timers.

How about documenting it in https://docs-cupy.chainer.org/en/stable/reference/difference.html?

(Note, to fully fix this issue, we need to implement CuPy version of `PyArray_FromAny`)
Got the same issue when split the prediction of Siamese Net to us Triplet loss.
Is there a workaround for this kind of usage?
```python
    pred = net(X)
    a_preds = [pred[i] for i in range(0, batch*3, 3)]
    p_preds = [pred[i] for i in range(1, batch*3, 3)]
    n_preds = [pred[i] for i in range(2, batch*3, 3)]
    
    xp = cuda.get_array_module(pred)
    a_preds = xp.asarray(a_preds)
    p_preds = xp.asarray(p_preds)
    n_preds = xp.asarray(n_preds)
    
    loss = F.triplet(a_preds, p_preds, n_preds)


If you want to convert a list of CuPy arrays with the same shape to a single CuPy array, you can use `xp.stack`.
(Note that this does not work with more complicated structure, e.g., a list of list of CuPy arrays.)
Now this behavior is documented in #1027.
Should we close this issue?
Let's close this issue after finishing discussion in #1030.
OK.

Commit message: add tests for a list of views

Patch to review:
diff --git a/cupy/core/core.pxd b/cupy/core/core.pxd
index 35d32f5591e..d064890f359 100644
--- a/cupy/core/core.pxd
+++ b/cupy/core/core.pxd
@@ -94,6 +94,14 @@ cdef class Indexer:
     cdef CPointer get_pointer(self)
 
 
+cdef tuple _get_concat_shape(object obj)
+cdef tuple _get_concat_shape_impl(object obj)
+cdef list _flatten_list(object obj)
+cdef ndarray _send_object_to_gpu(obj, dtype, str order, Py_ssize_t ndmin)
+cdef ndarray _send_numpy_array_list_to_gpu(
+    list arrays, dtype, const vector.vector[Py_ssize_t]& shape, str order,
+    Py_ssize_t ndmin)
+cpdef _alloc_pinned_memory(Py_ssize_t size)
 cpdef ndarray ascontiguousarray(ndarray a, dtype=*)
 cpdef Module compile_with_cache(str source, tuple options=*, arch=*,
                                 cachd_dir=*, prepend_cupy_headers=*)
diff --git a/cupy/core/core.pyx b/cupy/core/core.pyx
index b00552cbf15..6f3c6ee0a00 100644
--- a/cupy/core/core.pyx
+++ b/cupy/core/core.pyx
@@ -1695,10 +1695,12 @@ cpdef ndarray array(obj, dtype=None, bint copy=True, order='K',
     cdef Py_ssize_t ndim
     cdef ndarray a, src
     cdef size_t nbytes
+
     if subok:
         raise NotImplementedError
     if order is None:
         order = 'K'
+
     if isinstance(obj, ndarray):
         src = obj
         if dtype is None:
@@ -1708,7 +1710,7 @@ cpdef ndarray array(obj, dtype=None, bint copy=True, order='K',
         else:
             a = src.copy(order=order).astype(dtype, copy=False)
 
-        ndim = a._shape.size()
+        ndim = len(a._shape)
         if ndmin > ndim:
             if a is obj:
                 # When `copy` is False, `a` is same as `obj`.
@@ -1717,51 +1719,177 @@ cpdef ndarray array(obj, dtype=None, bint copy=True, order='K',
     elif hasattr(obj, '__cuda_array_interface__'):
         return array(_convert_object_with_cuda_array_interface(obj),
                      dtype, copy, order, subok, ndmin)
+    else:  # obj is sequence, numpy array, scalar or the other type of object
+        shape, elem_dtype, head = _get_concat_shape(obj)
+        if shape is not None and shape[-1] != 0:
+            # obj is a non-empty sequence of ndarrays which share same shape
+            # and dtype
+
+            # resulting array is C order unless 'F' is explicitly specified
+            # (i.e., it ignores order of element arrays in the sequence)
+            order = (
+                'F'
+                if order is not None and len(order) >= 1 and order[0] in 'Ff'
+                else 'C')
+            if elem_dtype.char not in '?bhilqBHILQefdFD':
+                raise ValueError('Unsupported dtype %s' % elem_dtype)
+            ndim = len(shape)
+            if ndmin > ndim:
+                shape = (1,) * (ndmin - ndim) + shape
+
+            if dtype is None:
+                dtype = elem_dtype
+            if isinstance(head, numpy.ndarray):  # obj is Seq[numpy.ndarray]
+                a = _send_numpy_array_list_to_gpu(
+                    obj, dtype, shape, order, ndmin)
+            elif isinstance(head, ndarray):  # obj is Seq[cupy.ndarray]
+                a = (_manipulation.concatenate_method(_flatten_list(obj), 0)
+                                  .reshape(shape)
+                                  .astype(dtype, order=order, copy=False))
+            else:  # should not be reached here
+                raise ValueError(
+                    "The elements of obj are unsupported type(s)")
+        else:
+            # obj is:
+            # - numpy array
+            # - scalar or sequence of scalar
+            # - empty sequence or sequence with elements whose shapes or
+            #   dtypes are unmatched
+            # - the other type of object
+
+            # fallback to numpy array and send it to GPU
+            a = _send_object_to_gpu(obj, dtype, order, ndmin)
+
+    return a
+
+
+cdef tuple _get_concat_shape(object obj):
+    # Returns a tuple of the following:
+    # 1. concatenated shape if it can be converted to a single CuPy array by
+    #    just concatenating it (i.e., the object is a (nested) sequence only
+    #    which contains NumPy/CuPy array(s) with same shape and dtype).
+    #    Returns None otherwise.
+    # 2. dtype if the object is an array
+    # 3. a first element in the object
+    return (_get_concat_shape_impl(obj) if isinstance(obj, (list, tuple))
+            else (None, None, None))
+
+
+cdef tuple _get_concat_shape_impl(object obj):
+    if isinstance(obj, (numpy.ndarray, ndarray)):
+        return (obj.shape, obj.dtype, obj)
+    elif isinstance(obj, (list, tuple)):
+        shape = None
+        dtype = None
+        head = None
+        for elem in obj:
+            # Find the head recursively if obj is a nested built-in list
+            elem_shape, elem_dtype, elem_head = _get_concat_shape_impl(elem)
+
+            # Use shape of the first element as the common shape.
+            if shape is None:
+                shape = elem_shape
+                dtype = elem_dtype
+                head = elem_head
+
+            # `elem` is not concatable or the shape and dtype does not match
+            # with siblings.
+            if (elem_shape is None or
+                    shape != elem_shape or
+                    dtype != elem_dtype):
+                return (None, None, obj)
+
+        return (
+            (len(obj),) + shape if shape is not None else (len(obj),),
+            dtype,
+            head)
+    else:  # scalar or object
+        return (None, None, obj)
+
+
+cdef list _flatten_list(object obj):
+    ret = []
+    if isinstance(obj, (list, tuple)):
+        for elem in obj:
+            ret += _flatten_list(elem)
+        return ret
     else:
-        if order is not None and len(order) >= 1 and order[0] in 'KAka':
-            if isinstance(obj, numpy.ndarray) and obj.flags.f_contiguous:
-                order = 'F'
-            else:
-                order = 'C'
-        a_cpu = numpy.array(obj, dtype=dtype, copy=False, order=order,
-                            ndmin=ndmin)
-        a_dtype = a_cpu.dtype
-        if a_dtype.char not in '?bhilqBHILQefdFD':
-            raise ValueError('Unsupported dtype %s' % a_dtype)
-        a = ndarray(a_cpu.shape, dtype=a_dtype, order=order)
-        if a_cpu.ndim == 0:
-            a.fill(a_cpu[()])
-            return a
-        nbytes = a.nbytes
-        stream = stream_module.get_current_stream()
+        return [obj]
 
-        mem = None
-        error = None
-        try:
-            mem = pinned_memory.alloc_pinned_memory(nbytes)
-        except CUDARuntimeError as e:
-            if e.status != runtime.errorMemoryAllocation:
-                raise
-            error = e
-
-        if mem is not None:
-            src_cpu = numpy.frombuffer(mem, a_dtype, a_cpu.size)
-            src_cpu[:] = a_cpu.ravel(order)
-            a.data.copy_from_host_async(ctypes.c_void_p(mem.ptr), nbytes)
-            pinned_memory._add_to_watch_list(stream.record(), mem)
+
+cdef ndarray _send_object_to_gpu(obj, dtype, str order, Py_ssize_t ndmin):
+    if order is not None and len(order) >= 1 and order[0] in 'KAka':
+        if isinstance(obj, numpy.ndarray) and obj.flags.f_contiguous:
+            order = 'F'
         else:
-            warnings.warn(
-                'Using synchronous transfer as pinned memory ({} bytes) '
-                'could not be allocated. '
-                'This generally occurs because of insufficient host memory. '
-                'The original error was: {}'.format(nbytes, error),
-                util.PerformanceWarning)
-            a.data.copy_from_host(
-                ctypes.c_void_p(a_cpu.__array_interface__['data'][0]), nbytes)
+            order = 'C'
+    a_cpu = numpy.array(obj, dtype=dtype, copy=False, order=order,
+                        ndmin=ndmin)
+    a_dtype = a_cpu.dtype
+    if a_dtype.char not in '?bhilqBHILQefdFD':
+        raise ValueError('Unsupported dtype %s' % a_dtype)
+    cdef vector.vector[Py_ssize_t] a_shape = (<object>a_cpu).shape
+    cdef ndarray a = ndarray(a_shape, dtype=a_dtype, order=order)
+    if a_cpu.ndim == 0:
+        a.fill(a_cpu[()])
+        return a
+    cdef Py_ssize_t nbytes = a.nbytes
+
+    stream = stream_module.get_current_stream()
+    cdef pinned_memory.PinnedMemoryPointer mem = _alloc_pinned_memory(nbytes)
+    if mem is not None:
+        src_cpu = numpy.frombuffer(mem, a_dtype, a_cpu.size)
+        src_cpu[:] = a_cpu.ravel(order)
+        a.data.copy_from_host_async(ctypes.c_void_p(mem.ptr), nbytes)
+        pinned_memory._add_to_watch_list(stream.record(), mem)
+    else:
+        a.data.copy_from_host(
+            ctypes.c_void_p(a_cpu.__array_interface__['data'][0]), nbytes)
 
     return a
 
 
+cdef ndarray _send_numpy_array_list_to_gpu(
+    list arrays, dtype, const vector.vector[Py_ssize_t]& shape, str order,
+        Py_ssize_t ndmin):
+    cdef ndarray a  # allocate it after pinned memory is secured
+    cdef size_t itemcount = internal.prod(shape)
+    cdef size_t nbytes = itemcount * get_dtype(dtype).itemsize
+
+    stream = stream_module.get_current_stream()
+    cdef pinned_memory.PinnedMemoryPointer mem = _alloc_pinned_memory(nbytes)
+    cdef size_t offset, length
+    if mem is not None:
+        # write concatenated arrays to the pinned memory directly
+        src_cpu = numpy.frombuffer(mem, dtype, itemcount).reshape(
+            shape, order=order)
+        numpy.concatenate([e[None] for e in arrays], 0, src_cpu)
+        a = ndarray(shape, dtype=dtype, order=order)
+        a.data.copy_from_host_async(ctypes.c_void_p(mem.ptr), nbytes)
+        pinned_memory._add_to_watch_list(stream.record(), mem)
+    else:
+        # fallback to numpy array and send it to GPU
+        a = _send_object_to_gpu(arrays, dtype, order, ndmin)
+
+    return a
+
+
+cpdef inline _alloc_pinned_memory(Py_ssize_t nbytes):
+    try:
+        return pinned_memory.alloc_pinned_memory(nbytes)
+    except CUDARuntimeError as e:
+        if e.status != runtime.errorMemoryAllocation:
+            raise
+        warnings.warn(
+            'Using synchronous transfer as pinned memory ({} bytes) '
+            'could not be allocated. '
+            'This generally occurs because of insufficient host memory. '
+            'The original error was: {}'.format(nbytes, e),
+            util.PerformanceWarning)
+
+    return None
+
+
 cpdef ndarray ascontiguousarray(ndarray a, dtype=None):
     if dtype is None:
         if a._c_contiguous:
diff --git a/cupy/testing/__init__.py b/cupy/testing/__init__.py
index 5f3da90adaa..d3c43ed5baa 100644
--- a/cupy/testing/__init__.py
+++ b/cupy/testing/__init__.py
@@ -33,6 +33,7 @@
 from cupy.testing.helper import numpy_cupy_array_almost_equal  # NOQA
 from cupy.testing.helper import numpy_cupy_array_almost_equal_nulp  # NOQA
 from cupy.testing.helper import numpy_cupy_array_equal  # NOQA
+from cupy.testing.helper import numpy_cupy_array_exactly_equal  # NOQA
 from cupy.testing.helper import numpy_cupy_array_less  # NOQA
 from cupy.testing.helper import numpy_cupy_array_list_equal  # NOQA
 from cupy.testing.helper import numpy_cupy_array_max_ulp  # NOQA
diff --git a/cupy/testing/array.py b/cupy/testing/array.py
index 211892c08ed..e32715b7e0c 100644
--- a/cupy/testing/array.py
+++ b/cupy/testing/array.py
@@ -1,3 +1,4 @@
+import ctypes
 import numpy.testing
 
 import cupy
@@ -128,6 +129,70 @@ def assert_array_list_equal(xlist, ylist, err_msg='', verbose=True):
             verbose=verbose)
 
 
+def _get_underlying_bytes(a):
+    if isinstance(a, cupy.ndarray):
+        # copy the array to host memory while preserving order
+        buf = ctypes.create_string_buffer(a.nbytes)
+        a.data.copy_to_host(ctypes.c_void_p(ctypes.addressof(buf)), a.nbytes)
+        arr = bytearray(a.nbytes)
+        arr[:] = buf.raw
+        return arr
+    else:
+        return ctypes.string_at(a.ctypes.data, a.nbytes)
+
+
+def assert_array_exactly_equal(x, y, err_msg='', verbose=True):
+    """Raises an AssertionError if two array_like objects are not equal in the
+    sense of its underlying memory layout.
+
+    Args:
+         x(numpy.ndarray or cupy.ndarray): The actual object to check.
+         y(numpy.ndarray or cupy.ndarray): The desired, expected object.
+         err_msg(str): The error message to be printed in case of failure.
+         verbose(bool): If ``True``, the conflicting values
+             are appended to the error message.
+    """
+    # checks an equality of the representation of two arrays
+    numpy.testing.assert_array_equal(
+        cupy.asnumpy(x), cupy.asnumpy(y), err_msg=err_msg,
+        verbose=verbose)
+
+    xbytes = _get_underlying_bytes(x)
+    ybytes = _get_underlying_bytes(y)
+    if xbytes != ybytes:
+        msg = ["Underlying arrays are not same:"]
+        if err_msg:
+            msg = [msg[0] + ' ' + err_msg]
+        if verbose:
+            msg.append(" x: {}".format(xbytes))
+            msg.append(" y: {}".format(ybytes))
+        raise AssertionError('\n'.join(msg))
+    if x.strides != y.strides:
+        msg = ["Strides are not equal:"]
+        if err_msg:
+            msg = [msg[0] + ' ' + err_msg]
+        if verbose:
+            msg.append(" x: {}".format(x.strides))
+            msg.append(" y: {}".format(y.strides))
+        raise AssertionError('\n'.join(msg))
+    if x.flags.c_contiguous != y.flags.c_contiguous:
+        msg = ["c_contiguous are not equal:"]
+        if err_msg:
+            msg = [msg[0] + ' ' + err_msg]
+        if verbose:
+            msg.append(" x: {}".format(x.c_contiguous))
+            msg.append(" y: {}".format(y.c_contiguous))
+        raise AssertionError('\n'.join(msg))
+    if x.flags.f_contiguous != y.flags.f_contiguous:
+        msg = ["f_contiguous are not equal:"]
+        if err_msg:
+            msg = [msg[0] + ' ' + err_msg]
+        if verbose:
+            msg.append(" x: {}".format(x.f_contiguous))
+            msg.append(" y: {}".format(y.f_contiguous))
+        raise AssertionError('\n'.join(msg))
+
+
 def assert_array_less(x, y, err_msg='', verbose=True):
     """Raises an AssertionError if array_like objects are not ordered by less than.
 
diff --git a/cupy/testing/helper.py b/cupy/testing/helper.py
index 0c287053960..4c4306dfe95 100644
--- a/cupy/testing/helper.py
+++ b/cupy/testing/helper.py
@@ -381,6 +381,52 @@ def check_func(x, y):
                            scipy_name)
 
 
+def numpy_cupy_array_exactly_equal(err_msg='', verbose=True, name='xp',
+                                   type_check=True, accept_error=False,
+                                   sp_name=None, scipy_name=None):
+    """Decorator that checks NumPy results and CuPy ones are equal in the
+    sense of its underlying memory layout.
+
+    Args:
+         err_msg(str): The error message to be printed in case of failure.
+         verbose(bool): If ``True``, the conflicting values are
+             appended to the error message.
+         name(str): Argument name whose value is either
+             ``numpy`` or ``cupy`` module.
+         type_check(bool): If ``True``, consistency of dtype is also checked.
+         accept_error(bool, Exception or tuple of Exception): Specify
+             acceptable errors. When both NumPy test and CuPy test raises the
+             same type of errors, and the type of the errors is specified with
+             this argument, the errors are ignored and not raised.
+             If it is ``True`` all error types are acceptable.
+             If it is ``False`` no error is acceptable.
+         sp_name(str or None): Argument name whose value is either
+             ``scipy.sparse`` or ``cupyx.scipy.sparse`` module. If ``None``, no
+             argument is given for the modules.
+         scipy_name(str or None): Argument name whose value is either ``scipy``
+             or ``cupyx.scipy`` module. If ``None``, no argument is given for
+             the modules.
+
+    Decorated test fixture is required to return the same arrays
+    in the sense of :func:`numpy_cupy_array_exactly_equal`
+    (except the type of array module) even if ``xp`` is ``numpy`` or ``cupy``.
+
+    .. seealso:: :func:`cupy.testing.numpy_cupy_array_exactly_equal`
+    """
+    def check_func(x, y):
+        if sp_name is not None:
+            import scipy.sparse
+            if cupyx.scipy.sparse.issparse(x):
+                x = x.A
+            if scipy.sparse.issparse(y):
+                y = y.A
+
+        array.assert_array_exactly_equal(x, y, err_msg, verbose)
+
+    return _make_decorator(check_func, name, type_check, accept_error, sp_name,
+                           scipy_name)
+
+
 def numpy_cupy_array_list_equal(
         err_msg='', verbose=True, name='xp', sp_name=None, scipy_name=None):
     """Decorator that checks the resulting lists of NumPy and CuPy's one are equal.
diff --git a/docs/source/reference/difference.rst b/docs/source/reference/difference.rst
index ecf8424912b..aee50dbcb1f 100644
--- a/docs/source/reference/difference.rst
+++ b/docs/source/reference/difference.rst
@@ -121,27 +121,6 @@ Data type of CuPy arrays cannot be non-numeric like strings and objects.
 See :ref:`overview` for details.
 
 
-Array creation from Python objects
-----------------------------------
-
-Currently, :func:`cupy.array` or :func:`cupy.asarray` cannot create an array from Python object containing CuPy array (e.g., a list of CuPy arrays).
-Use :func:`cupy.stack` instead.
-
-  >>> data_cpu = [np.arange(10), np.arange(10)]
-  >>> np.asarray(data_cpu)
-  array([[0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
-         [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]])
-
-  >>> data_gpu = [cupy.arange(10), cupy.arange(10)]
-  >>> cupy.asarray(data_gpu)
-  Traceback (most recent call last):
-    File "<stdin>", line 1, in <module>
-  ValueError: Unsupported dtype object
-  >>> cupy.stack(data_gpu)
-  array([[0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
-         [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]])
-
-
 Universal Functions only work with CuPy array or scalar
 -------------------------------------------------------
 
diff --git a/docs/source/reference/testing.rst b/docs/source/reference/testing.rst
index 2fc4834543f..650572c091c 100644
--- a/docs/source/reference/testing.rst
+++ b/docs/source/reference/testing.rst
@@ -43,6 +43,7 @@ between CuPy's functions and corresponding NumPy's ones.
    cupy.testing.numpy_cupy_array_almost_equal_nulp
    cupy.testing.numpy_cupy_array_max_ulp
    cupy.testing.numpy_cupy_array_equal
+   cupy.testing.numpy_cupy_array_exactly_equal
    cupy.testing.numpy_cupy_array_list_equal
    cupy.testing.numpy_cupy_array_less
    cupy.testing.numpy_cupy_raises
diff --git a/tests/cupy_tests/creation_tests/test_from_data.py b/tests/cupy_tests/creation_tests/test_from_data.py
index 8d47c80e680..f02b3085fd7 100644
--- a/tests/cupy_tests/creation_tests/test_from_data.py
+++ b/tests/cupy_tests/creation_tests/test_from_data.py
@@ -31,6 +31,62 @@ def test_array_from_numpy_broad_cast(self, xp, dtype, order):
         a = numpy.broadcast_to(a, (2, 3, 4))
         return xp.array(a, order=order)
 
+    @testing.for_orders('CFAK', name='src_order')
+    @testing.for_orders('CFAK', name='dst_order')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_exactly_equal()
+    def test_array_from_list_of_numpy(self, xp, dtype, src_order, dst_order):
+        # compares numpy.array(<list of numpy.ndarray>) with
+        # cupy.array(<list of numpy.ndarray>)
+        a = [
+            testing.shaped_arange((3, 4), numpy, dtype, src_order) + (12 * i)
+            for i in range(2)]
+        return xp.array(a, order=dst_order)
+
+    @testing.for_orders('CFAK', name='src_order')
+    @testing.for_orders('CFAK', name='dst_order')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_exactly_equal()
+    def test_array_from_list_of_numpy_view(self, xp, dtype, src_order,
+                                           dst_order):
+        # compares numpy.array(<list of numpy.ndarray>) with
+        # cupy.array(<list of numpy.ndarray>)
+
+        # create a list of view of ndarrays
+        a = [
+            (testing.shaped_arange((3, 8), numpy,
+                                   dtype, src_order) + (24 * i))[:, ::2]
+            for i in range(2)]
+        return xp.array(a, order=dst_order)
+
+    @testing.for_orders('CFAK', name='src_order')
+    @testing.for_orders('CFAK', name='dst_order')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_exactly_equal()
+    def test_array_from_list_of_cupy(self, xp, dtype, src_order, dst_order):
+        # compares numpy.array(<list of numpy.ndarray>) with
+        # cupy.array(<list of cupy.ndarray>)
+        a = [
+            testing.shaped_arange((3, 4), xp, dtype, src_order) + (12 * i)
+            for i in range(2)]
+        return xp.array(a, order=dst_order)
+
+    @testing.for_orders('CFAK', name='src_order')
+    @testing.for_orders('CFAK', name='dst_order')
+    @testing.for_all_dtypes()
+    @testing.numpy_cupy_array_exactly_equal()
+    def test_array_from_list_of_cupy_view(self, xp, dtype, src_order,
+                                          dst_order):
+        # compares numpy.array(<list of numpy.ndarray>) with
+        # cupy.array(<list of cupy.ndarray>)
+
+        # create a list of view of ndarrays
+        a = [
+            (testing.shaped_arange((3, 8), xp,
+                                   dtype, src_order) + (24 * i))[:, ::2]
+            for i in range(2)]
+        return xp.array(a, order=dst_order)
+
     @testing.for_orders('CFAK')
     @testing.for_all_dtypes()
     @testing.numpy_cupy_array_equal()


Code review comments:
diff_hunk: @@ -1717,51 +1719,177 @@ cpdef ndarray array(obj, dtype=None, bint copy=True, order='K',
     elif hasattr(obj, '__cuda_array_interface__'):
         return array(_convert_object_with_cuda_array_interface(obj),
                      dtype, copy, order, subok, ndmin)
+    else:  # obj is sequence, numpy array, scalar or the other type of object
+        shape, elem_dtype, head = _get_concat_shape(obj)
+        if shape is not None and shape[-1] != 0:
+            # obj is a non-empty sequence of ndarrays which share same shape
+            # and dtype
+
+            # resulting array is C order unless 'F' is explicitly specified
+            # (i.e., it ignores order of element arrays in the sequence)
+            order = (
+                'F'
+                if order is not None and len(order) >= 1 and order[0] in 'Ff'
+                else 'C')
+            if elem_dtype.char not in '?bhilqBHILQefdFD':
+                raise ValueError('Unsupported dtype %s' % elem_dtype)
+            ndim = len(shape)
+            if ndmin > ndim:
+                shape = (1,) * (ndmin - ndim) + shape
+
+            if dtype is None:
+                dtype = elem_dtype
+            if isinstance(head, numpy.ndarray):  # obj is Seq[numpy.ndarray]
+                a = _send_numpy_array_list_to_gpu(
+                    obj, dtype, shape, order, ndmin)
+            elif isinstance(head, ndarray):  # obj is Seq[cupy.ndarray]
+                a = (_manipulation.concatenate_method(_flatten_list(obj), 0)
+                                  .reshape(shape)
+                                  .astype(dtype, order=order, copy=False))
+            else:  # should not be reached here
+                raise ValueError(
+                    "The elements of obj are unsupported type(s)")
+        else:
+            # obj is:
+            # - numpy array
+            # - scalar or sequence of scalar
+            # - empty sequence or sequence with elements whose shapes or
+            #   dtypes are unmatched
+            # - the other type of object
+
+            # fallback to numpy array and send it to GPU
+            a = _send_object_to_gpu(obj, dtype, order, ndmin)
+
+    return a
+
+
+cdef tuple _get_concat_shape(object obj):
+    # Returns a tuple of the following:
+    # 1. concatenated shape if it can be converted to a single CuPy array by
+    #    just concatenating it (i.e., the object is a (nested) sequence only
+    #    which contains NumPy/CuPy array(s) with same shape and dtype).
+    #    Returns None otherwise.
+    # 2. dtype if the object is an array
+    # 3. a first element in the object
+    return (_get_concat_shape_impl(obj) if isinstance(obj, (list, tuple))
+            else (None, None, None))
+
+
+cdef tuple _get_concat_shape_impl(object obj):
+    if isinstance(obj, (numpy.ndarray, ndarray)):
+        return (obj.shape, obj.dtype, obj)
+    elif isinstance(obj, (list, tuple)):
+        shape = None
+        dtype = None
+        head = None
+        for elem in obj:
+            # Find the head recursively if obj is a nested built-in list
+            elem_shape, elem_dtype, elem_head = _get_concat_shape_impl(elem)
+
+            # Use shape of the first element as the common shape.
+            if shape is None:
+                shape = elem_shape
+                dtype = elem_dtype
+                head = elem_head
+
+            # `elem` is not concatable or the shape and dtype does not match
+            # with siblings.
+            if (elem_shape is None or
+                    shape != elem_shape or
+                    dtype != elem_dtype):
+                return (None, None, obj)
+
+        return (
+            (len(obj),) + shape if shape is not None else (len(obj),),
+            dtype,
+            head)
+    else:  # scalar or object
+        return (None, None, obj)
+
+
+cdef list _flatten_list(object obj):
+    ret = []
+    if isinstance(obj, (list, tuple)):
+        for elem in obj:
+            ret += _flatten_list(elem)
+        return ret
     else:
-        if order is not None and len(order) >= 1 and order[0] in 'KAka':
-            if isinstance(obj, numpy.ndarray) and obj.flags.f_contiguous:
-                order = 'F'
-            else:
-                order = 'C'
-        a_cpu = numpy.array(obj, dtype=dtype, copy=False, order=order,
-                            ndmin=ndmin)
-        a_dtype = a_cpu.dtype
-        if a_dtype.char not in '?bhilqBHILQefdFD':
-            raise ValueError('Unsupported dtype %s' % a_dtype)
-        a = ndarray(a_cpu.shape, dtype=a_dtype, order=order)
-        if a_cpu.ndim == 0:
-            a.fill(a_cpu[()])
-            return a
-        nbytes = a.nbytes
-        stream = stream_module.get_current_stream()
+        return [obj]
 
-        mem = None
-        error = None
-        try:
-            mem = pinned_memory.alloc_pinned_memory(nbytes)
-        except CUDARuntimeError as e:
-            if e.status != runtime.errorMemoryAllocation:
-                raise
-            error = e
-
-        if mem is not None:
-            src_cpu = numpy.frombuffer(mem, a_dtype, a_cpu.size)
-            src_cpu[:] = a_cpu.ravel(order)
-            a.data.copy_from_host_async(ctypes.c_void_p(mem.ptr), nbytes)
-            pinned_memory._add_to_watch_list(stream.record(), mem)
+
+cdef ndarray _send_object_to_gpu(obj, dtype, str order, Py_ssize_t ndmin):
+    if order is not None and len(order) >= 1 and order[0] in 'KAka':
+        if isinstance(obj, numpy.ndarray) and obj.flags.f_contiguous:
+            order = 'F'
         else:
-            warnings.warn(
-                'Using synchronous transfer as pinned memory ({} bytes) '
-                'could not be allocated. '
-                'This generally occurs because of insufficient host memory. '
-                'The original error was: {}'.format(nbytes, error),
-                util.PerformanceWarning)
-            a.data.copy_from_host(
-                ctypes.c_void_p(a_cpu.__array_interface__['data'][0]), nbytes)
+            order = 'C'
+    a_cpu = numpy.array(obj, dtype=dtype, copy=False, order=order,
+                        ndmin=ndmin)
+    a_dtype = a_cpu.dtype
+    if a_dtype.char not in '?bhilqBHILQefdFD':
+        raise ValueError('Unsupported dtype %s' % a_dtype)
+    cdef vector.vector[Py_ssize_t] a_shape = (<object>a_cpu).shape
path: cupy/core/core.pyx
text: @user1:
The cast `<object>` is not needed?

diff_hunk: @@ -1708,7 +1710,7 @@ cpdef ndarray array(obj, dtype=None, bint copy=True, order='K',
         else:
             a = src.copy(order=order).astype(dtype, copy=False)
 
-        ndim = a._shape.size()
+        ndim = len(a._shape)
path: cupy/core/core.pyx
text: @user1:
Is there any reason of this change? Is `len` faster?

@author:
It is because I saw that the compiler complains `a._shape` doesn't have `size()` method (maybe it fails to infer `_shape` as a tuple). I now checked again and `.size()` works... It is strange. Anyway, I'll fix it.

diff_hunk: @@ -1717,51 +1719,177 @@ cpdef ndarray array(obj, dtype=None, bint copy=True, order='K',
     elif hasattr(obj, '__cuda_array_interface__'):
         return array(_convert_object_with_cuda_array_interface(obj),
                      dtype, copy, order, subok, ndmin)
+    else:  # obj is sequence, numpy array, scalar or the other type of object
+        shape, elem_dtype, head = _get_concat_shape(obj)
+        if shape is not None and shape[-1] != 0:
+            # obj is a non-empty sequence of ndarrays which share same shape
+            # and dtype
+
+            # resulting array is C order unless 'F' is explicitly specified
+            # (i.e., it ignores order of element arrays in the sequence)
+            order = (
+                'F'
+                if order is not None and len(order) >= 1 and order[0] in 'Ff'
+                else 'C')
+            if elem_dtype.char not in '?bhilqBHILQefdFD':
+                raise ValueError('Unsupported dtype %s' % elem_dtype)
+            ndim = len(shape)
+            if ndmin > ndim:
+                shape = (1,) * (ndmin - ndim) + shape
+
+            if dtype is None:
+                dtype = elem_dtype
+            if isinstance(head, numpy.ndarray):  # obj is Seq[numpy.ndarray]
+                a = _send_numpy_array_list_to_gpu(
+                    obj, dtype, shape, order, ndmin)
+            elif isinstance(head, ndarray):  # obj is Seq[cupy.ndarray]
+                a = (_manipulation.concatenate_method(_flatten_list(obj), 0)
+                                  .reshape(shape)
+                                  .astype(dtype, order=order, copy=False))
+            else:  # should not be reached here
+                raise ValueError(
+                    "The elements of obj are unsupported type(s)")
+        else:
+            # obj is:
+            # - numpy array
+            # - scalar or sequence of scalar
+            # - empty sequence or sequence with elements whose shapes or
+            #   dtypes are unmatched
+            # - the other type of object
+
+            # fallback to numpy array and send it to GPU
+            a = _send_object_to_gpu(obj, dtype, order, ndmin)
+
+    return a
+
+
+cdef tuple _get_concat_shape(object obj):
+    # Returns a tuple of the following:
+    # 1. concatenated shape if it can be converted to a single CuPy array by
+    #    just concatenating it (i.e., the object is a (nested) sequence only
+    #    which contains NumPy/CuPy array(s) with same shape and dtype).
+    #    Returns None otherwise.
+    # 2. dtype if the object is an array
+    # 3. a first element in the object
path: cupy/core/core.pyx
text: @user1:
How about using "item" instead of "element"?
"Element" more commonly refers to a scalar in arrays.

diff_hunk: @@ -1717,51 +1719,177 @@ cpdef ndarray array(obj, dtype=None, bint copy=True, order='K',
     elif hasattr(obj, '__cuda_array_interface__'):
         return array(_convert_object_with_cuda_array_interface(obj),
                      dtype, copy, order, subok, ndmin)
+    else:  # obj is sequence, numpy array, scalar or the other type of object
+        shape, elem_dtype, head = _get_concat_shape(obj)
+        if shape is not None and shape[-1] != 0:
+            # obj is a non-empty sequence of ndarrays which share same shape
+            # and dtype
+
+            # resulting array is C order unless 'F' is explicitly specified
+            # (i.e., it ignores order of element arrays in the sequence)
+            order = (
+                'F'
+                if order is not None and len(order) >= 1 and order[0] in 'Ff'
+                else 'C')
+            if elem_dtype.char not in '?bhilqBHILQefdFD':
+                raise ValueError('Unsupported dtype %s' % elem_dtype)
+            ndim = len(shape)
+            if ndmin > ndim:
+                shape = (1,) * (ndmin - ndim) + shape
+
+            if dtype is None:
+                dtype = elem_dtype
+            if isinstance(head, numpy.ndarray):  # obj is Seq[numpy.ndarray]
+                a = _send_numpy_array_list_to_gpu(
+                    obj, dtype, shape, order, ndmin)
+            elif isinstance(head, ndarray):  # obj is Seq[cupy.ndarray]
+                a = (_manipulation.concatenate_method(_flatten_list(obj), 0)
+                                  .reshape(shape)
+                                  .astype(dtype, order=order, copy=False))
+            else:  # should not be reached here
+                raise ValueError(
+                    "The elements of obj are unsupported type(s)")
+        else:
+            # obj is:
+            # - numpy array
+            # - scalar or sequence of scalar
+            # - empty sequence or sequence with elements whose shapes or
+            #   dtypes are unmatched
+            # - the other type of object
+
+            # fallback to numpy array and send it to GPU
+            a = _send_object_to_gpu(obj, dtype, order, ndmin)
+
+    return a
+
+
+cdef tuple _get_concat_shape(object obj):
+    # Returns a tuple of the following:
+    # 1. concatenated shape if it can be converted to a single CuPy array by
+    #    just concatenating it (i.e., the object is a (nested) sequence only
+    #    which contains NumPy/CuPy array(s) with same shape and dtype).
+    #    Returns None otherwise.
+    # 2. dtype if the object is an array
+    # 3. a first element in the object
+    return (_get_concat_shape_impl(obj) if isinstance(obj, (list, tuple))
+            else (None, None, None))
+
+
+cdef tuple _get_concat_shape_impl(object obj):
+    if isinstance(obj, (numpy.ndarray, ndarray)):
+        return (obj.shape, obj.dtype, obj)
+    elif isinstance(obj, (list, tuple)):
+        shape = None
+        dtype = None
+        head = None
+        for elem in obj:
+            # Find the head recursively if obj is a nested built-in list
+            elem_shape, elem_dtype, elem_head = _get_concat_shape_impl(elem)
+
+            # Use shape of the first element as the common shape.
+            if shape is None:
+                shape = elem_shape
+                dtype = elem_dtype
+                head = elem_head
+
+            # `elem` is not concatable or the shape and dtype does not match
+            # with siblings.
+            if (elem_shape is None or
+                    shape != elem_shape or
+                    dtype != elem_dtype):
+                return (None, None, obj)
+
+        return (
+            (len(obj),) + shape if shape is not None else (len(obj),),
+            dtype,
+            head)
+    else:  # scalar or object
+        return (None, None, obj)
+
+
+cdef list _flatten_list(object obj):
+    ret = []
+    if isinstance(obj, (list, tuple)):
+        for elem in obj:
+            ret += _flatten_list(elem)
+        return ret
     else:
-        if order is not None and len(order) >= 1 and order[0] in 'KAka':
-            if isinstance(obj, numpy.ndarray) and obj.flags.f_contiguous:
-                order = 'F'
-            else:
-                order = 'C'
-        a_cpu = numpy.array(obj, dtype=dtype, copy=False, order=order,
-                            ndmin=ndmin)
-        a_dtype = a_cpu.dtype
-        if a_dtype.char not in '?bhilqBHILQefdFD':
-            raise ValueError('Unsupported dtype %s' % a_dtype)
-        a = ndarray(a_cpu.shape, dtype=a_dtype, order=order)
-        if a_cpu.ndim == 0:
-            a.fill(a_cpu[()])
-            return a
-        nbytes = a.nbytes
-        stream = stream_module.get_current_stream()
+        return [obj]
 
-        mem = None
-        error = None
-        try:
-            mem = pinned_memory.alloc_pinned_memory(nbytes)
-        except CUDARuntimeError as e:
-            if e.status != runtime.errorMemoryAllocation:
-                raise
-            error = e
-
-        if mem is not None:
-            src_cpu = numpy.frombuffer(mem, a_dtype, a_cpu.size)
-            src_cpu[:] = a_cpu.ravel(order)
-            a.data.copy_from_host_async(ctypes.c_void_p(mem.ptr), nbytes)
-            pinned_memory._add_to_watch_list(stream.record(), mem)
+
+cdef ndarray _send_object_to_gpu(obj, dtype, str order, Py_ssize_t ndmin):
+    if order is not None and len(order) >= 1 and order[0] in 'KAka':
+        if isinstance(obj, numpy.ndarray) and obj.flags.f_contiguous:
+            order = 'F'
         else:
-            warnings.warn(
-                'Using synchronous transfer as pinned memory ({} bytes) '
-                'could not be allocated. '
-                'This generally occurs because of insufficient host memory. '
-                'The original error was: {}'.format(nbytes, error),
-                util.PerformanceWarning)
-            a.data.copy_from_host(
-                ctypes.c_void_p(a_cpu.__array_interface__['data'][0]), nbytes)
+            order = 'C'
+    a_cpu = numpy.array(obj, dtype=dtype, copy=False, order=order,
+                        ndmin=ndmin)
+    a_dtype = a_cpu.dtype
+    if a_dtype.char not in '?bhilqBHILQefdFD':
+        raise ValueError('Unsupported dtype %s' % a_dtype)
+    cdef vector.vector[Py_ssize_t] a_shape = (<object>a_cpu).shape
+    cdef ndarray a = ndarray(a_shape, dtype=a_dtype, order=order)
+    if a_cpu.ndim == 0:
+        a.fill(a_cpu[()])
+        return a
+    cdef Py_ssize_t nbytes = a.nbytes
+
+    stream = stream_module.get_current_stream()
+    cdef pinned_memory.PinnedMemoryPointer mem = _alloc_pinned_memory(nbytes)
+    if mem is not None:
+        src_cpu = numpy.frombuffer(mem, a_dtype, a_cpu.size)
+        src_cpu[:] = a_cpu.ravel(order)
+        a.data.copy_from_host_async(ctypes.c_void_p(mem.ptr), nbytes)
+        pinned_memory._add_to_watch_list(stream.record(), mem)
+    else:
+        a.data.copy_from_host(
+            ctypes.c_void_p(a_cpu.__array_interface__['data'][0]), nbytes)
 
     return a
 
 
+cdef ndarray _send_numpy_array_list_to_gpu(
+    list arrays, dtype, const vector.vector[Py_ssize_t]& shape, str order,
+        Py_ssize_t ndmin):
+    cdef ndarray a  # allocate it after pinned memory is secured
+    cdef size_t itemcount = internal.prod(shape)
+    cdef size_t nbytes = itemcount * get_dtype(dtype).itemsize
+
+    stream = stream_module.get_current_stream()
+    cdef pinned_memory.PinnedMemoryPointer mem = _alloc_pinned_memory(nbytes)
+    cdef size_t offset, length
+    if mem is not None:
+        # write concatenated arrays to the pinned memory directly
+        src_cpu = numpy.frombuffer(mem, dtype, itemcount).reshape(
+            shape, order=order)
+        numpy.concatenate([e[None] for e in arrays], 0, src_cpu)
+        a = ndarray(shape, dtype=dtype, order=order)
+        a.data.copy_from_host_async(ctypes.c_void_p(mem.ptr), nbytes)
+        pinned_memory._add_to_watch_list(stream.record(), mem)
+    else:
+        # fallback to numpy array and send it to GPU
+        a = _send_object_to_gpu(arrays, dtype, order, ndmin)
+
+    return a
+
+
+cpdef inline _alloc_pinned_memory(Py_ssize_t nbytes):
path: cupy/core/core.pyx
text: @user1:
Isn't it possible to make this function do the transfer as well, not only allocation?
The warning looks out of place.

@author:
Now this is used by both `_send_object_to_gpu` and `_send_numpy_array_list_to_gpu` which have different transfer logics. One idea is that move the warning to the above functions. Do you have any ideas?

@author:
I remembered that it is hard to move to other places because the message takes `e` as an argument.

@author:
Renamed to `_alloc_async_tranfer_buffer` for clarity.

diff_hunk: @@ -94,6 +94,14 @@ cdef class Indexer:
     cdef CPointer get_pointer(self)
 
 
+cdef tuple _get_concat_shape(object obj)
path: cupy/core/core.pxd
text: @user1:
I think all of the declarations in `.pxd` are not needed.

@author:
Thanks. I'll remove them.

diff_hunk: @@ -1717,51 +1719,177 @@ cpdef ndarray array(obj, dtype=None, bint copy=True, order='K',
     elif hasattr(obj, '__cuda_array_interface__'):
         return array(_convert_object_with_cuda_array_interface(obj),
                      dtype, copy, order, subok, ndmin)
+    else:  # obj is sequence, numpy array, scalar or the other type of object
+        shape, elem_dtype, head = _get_concat_shape(obj)
+        if shape is not None and shape[-1] != 0:
+            # obj is a non-empty sequence of ndarrays which share same shape
+            # and dtype
+
+            # resulting array is C order unless 'F' is explicitly specified
+            # (i.e., it ignores order of element arrays in the sequence)
+            order = (
+                'F'
+                if order is not None and len(order) >= 1 and order[0] in 'Ff'
+                else 'C')
+            if elem_dtype.char not in '?bhilqBHILQefdFD':
+                raise ValueError('Unsupported dtype %s' % elem_dtype)
+            ndim = len(shape)
+            if ndmin > ndim:
+                shape = (1,) * (ndmin - ndim) + shape
+
+            if dtype is None:
+                dtype = elem_dtype
+            if isinstance(head, numpy.ndarray):  # obj is Seq[numpy.ndarray]
+                a = _send_numpy_array_list_to_gpu(
+                    obj, dtype, shape, order, ndmin)
+            elif isinstance(head, ndarray):  # obj is Seq[cupy.ndarray]
+                a = (_manipulation.concatenate_method(_flatten_list(obj), 0)
+                                  .reshape(shape)
+                                  .astype(dtype, order=order, copy=False))
+            else:  # should not be reached here
+                raise ValueError(
+                    "The elements of obj are unsupported type(s)")
+        else:
+            # obj is:
+            # - numpy array
+            # - scalar or sequence of scalar
+            # - empty sequence or sequence with elements whose shapes or
+            #   dtypes are unmatched
+            # - the other type of object
+
+            # fallback to numpy array and send it to GPU
+            a = _send_object_to_gpu(obj, dtype, order, ndmin)
+
+    return a
+
+
+cdef tuple _get_concat_shape(object obj):
+    # Returns a tuple of the following:
+    # 1. concatenated shape if it can be converted to a single CuPy array by
+    #    just concatenating it (i.e., the object is a (nested) sequence only
+    #    which contains NumPy/CuPy array(s) with same shape and dtype).
+    #    Returns None otherwise.
+    # 2. dtype if the object is an array
+    # 3. a first element in the object
path: cupy/core/core.pyx
text: @user1:
It looks only its type is used.
How about returning the type instead?

diff_hunk: @@ -1717,51 +1719,177 @@ cpdef ndarray array(obj, dtype=None, bint copy=True, order='K',
     elif hasattr(obj, '__cuda_array_interface__'):
         return array(_convert_object_with_cuda_array_interface(obj),
                      dtype, copy, order, subok, ndmin)
+    else:  # obj is sequence, numpy array, scalar or the other type of object
+        shape, elem_dtype, head = _get_concat_shape(obj)
+        if shape is not None and shape[-1] != 0:
+            # obj is a non-empty sequence of ndarrays which share same shape
+            # and dtype
+
+            # resulting array is C order unless 'F' is explicitly specified
+            # (i.e., it ignores order of element arrays in the sequence)
+            order = (
+                'F'
+                if order is not None and len(order) >= 1 and order[0] in 'Ff'
+                else 'C')
+            if elem_dtype.char not in '?bhilqBHILQefdFD':
+                raise ValueError('Unsupported dtype %s' % elem_dtype)
+            ndim = len(shape)
+            if ndmin > ndim:
+                shape = (1,) * (ndmin - ndim) + shape
+
+            if dtype is None:
+                dtype = elem_dtype
+            if isinstance(head, numpy.ndarray):  # obj is Seq[numpy.ndarray]
+                a = _send_numpy_array_list_to_gpu(
+                    obj, dtype, shape, order, ndmin)
+            elif isinstance(head, ndarray):  # obj is Seq[cupy.ndarray]
+                a = (_manipulation.concatenate_method(_flatten_list(obj), 0)
+                                  .reshape(shape)
+                                  .astype(dtype, order=order, copy=False))
+            else:  # should not be reached here
+                raise ValueError(
+                    "The elements of obj are unsupported type(s)")
+        else:
+            # obj is:
+            # - numpy array
+            # - scalar or sequence of scalar
+            # - empty sequence or sequence with elements whose shapes or
+            #   dtypes are unmatched
+            # - the other type of object
+
+            # fallback to numpy array and send it to GPU
+            a = _send_object_to_gpu(obj, dtype, order, ndmin)
+
+    return a
+
+
+cdef tuple _get_concat_shape(object obj):
+    # Returns a tuple of the following:
+    # 1. concatenated shape if it can be converted to a single CuPy array by
+    #    just concatenating it (i.e., the object is a (nested) sequence only
+    #    which contains NumPy/CuPy array(s) with same shape and dtype).
+    #    Returns None otherwise.
+    # 2. dtype if the object is an array
+    # 3. a first element in the object
+    return (_get_concat_shape_impl(obj) if isinstance(obj, (list, tuple))
+            else (None, None, None))
+
+
+cdef tuple _get_concat_shape_impl(object obj):
+    if isinstance(obj, (numpy.ndarray, ndarray)):
+        return (obj.shape, obj.dtype, obj)
+    elif isinstance(obj, (list, tuple)):
+        shape = None
+        dtype = None
+        head = None
+        for elem in obj:
+            # Find the head recursively if obj is a nested built-in list
+            elem_shape, elem_dtype, elem_head = _get_concat_shape_impl(elem)
+
+            # Use shape of the first element as the common shape.
+            if shape is None:
+                shape = elem_shape
+                dtype = elem_dtype
+                head = elem_head
+
+            # `elem` is not concatable or the shape and dtype does not match
+            # with siblings.
+            if (elem_shape is None or
+                    shape != elem_shape or
+                    dtype != elem_dtype):
+                return (None, None, obj)
+
+        return (
+            (len(obj),) + shape if shape is not None else (len(obj),),
+            dtype,
+            head)
+    else:  # scalar or object
+        return (None, None, obj)
+
+
+cdef list _flatten_list(object obj):
+    ret = []
+    if isinstance(obj, (list, tuple)):
+        for elem in obj:
+            ret += _flatten_list(elem)
+        return ret
     else:
-        if order is not None and len(order) >= 1 and order[0] in 'KAka':
-            if isinstance(obj, numpy.ndarray) and obj.flags.f_contiguous:
-                order = 'F'
-            else:
-                order = 'C'
-        a_cpu = numpy.array(obj, dtype=dtype, copy=False, order=order,
-                            ndmin=ndmin)
-        a_dtype = a_cpu.dtype
-        if a_dtype.char not in '?bhilqBHILQefdFD':
-            raise ValueError('Unsupported dtype %s' % a_dtype)
-        a = ndarray(a_cpu.shape, dtype=a_dtype, order=order)
-        if a_cpu.ndim == 0:
-            a.fill(a_cpu[()])
-            return a
-        nbytes = a.nbytes
-        stream = stream_module.get_current_stream()
+        return [obj]
 
-        mem = None
-        error = None
-        try:
-            mem = pinned_memory.alloc_pinned_memory(nbytes)
-        except CUDARuntimeError as e:
-            if e.status != runtime.errorMemoryAllocation:
-                raise
-            error = e
-
-        if mem is not None:
-            src_cpu = numpy.frombuffer(mem, a_dtype, a_cpu.size)
-            src_cpu[:] = a_cpu.ravel(order)
-            a.data.copy_from_host_async(ctypes.c_void_p(mem.ptr), nbytes)
-            pinned_memory._add_to_watch_list(stream.record(), mem)
+
+cdef ndarray _send_object_to_gpu(obj, dtype, str order, Py_ssize_t ndmin):
+    if order is not None and len(order) >= 1 and order[0] in 'KAka':
+        if isinstance(obj, numpy.ndarray) and obj.flags.f_contiguous:
+            order = 'F'
         else:
-            warnings.warn(
-                'Using synchronous transfer as pinned memory ({} bytes) '
-                'could not be allocated. '
-                'This generally occurs because of insufficient host memory. '
-                'The original error was: {}'.format(nbytes, error),
-                util.PerformanceWarning)
-            a.data.copy_from_host(
-                ctypes.c_void_p(a_cpu.__array_interface__['data'][0]), nbytes)
+            order = 'C'
+    a_cpu = numpy.array(obj, dtype=dtype, copy=False, order=order,
+                        ndmin=ndmin)
+    a_dtype = a_cpu.dtype
+    if a_dtype.char not in '?bhilqBHILQefdFD':
+        raise ValueError('Unsupported dtype %s' % a_dtype)
+    cdef vector.vector[Py_ssize_t] a_shape = (<object>a_cpu).shape
+    cdef ndarray a = ndarray(a_shape, dtype=a_dtype, order=order)
+    if a_cpu.ndim == 0:
+        a.fill(a_cpu[()])
path: cupy/core/core.pyx
text: @user1:
`[()]` is not needed?

@author:
Hmm, it was introduced about two years ago in #157. I cannot judge this indexing (`()`) is redundant or not. What do you think?

@author:
As an offline discussion, we found it can be replaced with `a.fill(a_cpu)` because `fill` calls `a_cpu.item()` internally.

diff_hunk: @@ -1717,51 +1719,177 @@ cpdef ndarray array(obj, dtype=None, bint copy=True, order='K',
     elif hasattr(obj, '__cuda_array_interface__'):
         return array(_convert_object_with_cuda_array_interface(obj),
                      dtype, copy, order, subok, ndmin)
+    else:  # obj is sequence, numpy array, scalar or the other type of object
+        shape, elem_dtype, head = _get_concat_shape(obj)
+        if shape is not None and shape[-1] != 0:
+            # obj is a non-empty sequence of ndarrays which share same shape
+            # and dtype
+
+            # resulting array is C order unless 'F' is explicitly specified
+            # (i.e., it ignores order of element arrays in the sequence)
+            order = (
+                'F'
+                if order is not None and len(order) >= 1 and order[0] in 'Ff'
+                else 'C')
+            if elem_dtype.char not in '?bhilqBHILQefdFD':
+                raise ValueError('Unsupported dtype %s' % elem_dtype)
+            ndim = len(shape)
+            if ndmin > ndim:
+                shape = (1,) * (ndmin - ndim) + shape
+
+            if dtype is None:
+                dtype = elem_dtype
+            if isinstance(head, numpy.ndarray):  # obj is Seq[numpy.ndarray]
+                a = _send_numpy_array_list_to_gpu(
+                    obj, dtype, shape, order, ndmin)
+            elif isinstance(head, ndarray):  # obj is Seq[cupy.ndarray]
+                a = (_manipulation.concatenate_method(_flatten_list(obj), 0)
+                                  .reshape(shape)
+                                  .astype(dtype, order=order, copy=False))
+            else:  # should not be reached here
+                raise ValueError(
+                    "The elements of obj are unsupported type(s)")
+        else:
+            # obj is:
+            # - numpy array
+            # - scalar or sequence of scalar
+            # - empty sequence or sequence with elements whose shapes or
+            #   dtypes are unmatched
+            # - the other type of object
+
+            # fallback to numpy array and send it to GPU
+            a = _send_object_to_gpu(obj, dtype, order, ndmin)
+
+    return a
+
+
+cdef tuple _get_concat_shape(object obj):
+    # Returns a tuple of the following:
+    # 1. concatenated shape if it can be converted to a single CuPy array by
+    #    just concatenating it (i.e., the object is a (nested) sequence only
+    #    which contains NumPy/CuPy array(s) with same shape and dtype).
+    #    Returns None otherwise.
+    # 2. dtype if the object is an array
+    # 3. a first element in the object
+    return (_get_concat_shape_impl(obj) if isinstance(obj, (list, tuple))
+            else (None, None, None))
+
+
+cdef tuple _get_concat_shape_impl(object obj):
+    if isinstance(obj, (numpy.ndarray, ndarray)):
+        return (obj.shape, obj.dtype, obj)
+    elif isinstance(obj, (list, tuple)):
+        shape = None
+        dtype = None
+        head = None
+        for elem in obj:
+            # Find the head recursively if obj is a nested built-in list
+            elem_shape, elem_dtype, elem_head = _get_concat_shape_impl(elem)
+
+            # Use shape of the first element as the common shape.
+            if shape is None:
+                shape = elem_shape
+                dtype = elem_dtype
+                head = elem_head
+
+            # `elem` is not concatable or the shape and dtype does not match
+            # with siblings.
+            if (elem_shape is None or
+                    shape != elem_shape or
+                    dtype != elem_dtype):
+                return (None, None, obj)
+
+        return (
+            (len(obj),) + shape if shape is not None else (len(obj),),
+            dtype,
+            head)
+    else:  # scalar or object
+        return (None, None, obj)
+
+
+cdef list _flatten_list(object obj):
+    ret = []
+    if isinstance(obj, (list, tuple)):
+        for elem in obj:
+            ret += _flatten_list(elem)
+        return ret
     else:
-        if order is not None and len(order) >= 1 and order[0] in 'KAka':
-            if isinstance(obj, numpy.ndarray) and obj.flags.f_contiguous:
-                order = 'F'
-            else:
-                order = 'C'
-        a_cpu = numpy.array(obj, dtype=dtype, copy=False, order=order,
-                            ndmin=ndmin)
-        a_dtype = a_cpu.dtype
-        if a_dtype.char not in '?bhilqBHILQefdFD':
-            raise ValueError('Unsupported dtype %s' % a_dtype)
-        a = ndarray(a_cpu.shape, dtype=a_dtype, order=order)
-        if a_cpu.ndim == 0:
-            a.fill(a_cpu[()])
-            return a
-        nbytes = a.nbytes
-        stream = stream_module.get_current_stream()
+        return [obj]
 
-        mem = None
-        error = None
-        try:
-            mem = pinned_memory.alloc_pinned_memory(nbytes)
-        except CUDARuntimeError as e:
-            if e.status != runtime.errorMemoryAllocation:
-                raise
-            error = e
-
-        if mem is not None:
-            src_cpu = numpy.frombuffer(mem, a_dtype, a_cpu.size)
-            src_cpu[:] = a_cpu.ravel(order)
-            a.data.copy_from_host_async(ctypes.c_void_p(mem.ptr), nbytes)
-            pinned_memory._add_to_watch_list(stream.record(), mem)
+
+cdef ndarray _send_object_to_gpu(obj, dtype, str order, Py_ssize_t ndmin):
+    if order is not None and len(order) >= 1 and order[0] in 'KAka':
+        if isinstance(obj, numpy.ndarray) and obj.flags.f_contiguous:
+            order = 'F'
         else:
-            warnings.warn(
-                'Using synchronous transfer as pinned memory ({} bytes) '
-                'could not be allocated. '
-                'This generally occurs because of insufficient host memory. '
-                'The original error was: {}'.format(nbytes, error),
-                util.PerformanceWarning)
-            a.data.copy_from_host(
-                ctypes.c_void_p(a_cpu.__array_interface__['data'][0]), nbytes)
+            order = 'C'
+    a_cpu = numpy.array(obj, dtype=dtype, copy=False, order=order,
+                        ndmin=ndmin)
+    a_dtype = a_cpu.dtype
+    if a_dtype.char not in '?bhilqBHILQefdFD':
+        raise ValueError('Unsupported dtype %s' % a_dtype)
+    cdef vector.vector[Py_ssize_t] a_shape = (<object>a_cpu).shape
+    cdef ndarray a = ndarray(a_shape, dtype=a_dtype, order=order)
+    if a_cpu.ndim == 0:
+        a.fill(a_cpu[()])
+        return a
+    cdef Py_ssize_t nbytes = a.nbytes
+
+    stream = stream_module.get_current_stream()
+    cdef pinned_memory.PinnedMemoryPointer mem = _alloc_pinned_memory(nbytes)
+    if mem is not None:
+        src_cpu = numpy.frombuffer(mem, a_dtype, a_cpu.size)
+        src_cpu[:] = a_cpu.ravel(order)
+        a.data.copy_from_host_async(ctypes.c_void_p(mem.ptr), nbytes)
+        pinned_memory._add_to_watch_list(stream.record(), mem)
+    else:
+        a.data.copy_from_host(
+            ctypes.c_void_p(a_cpu.__array_interface__['data'][0]), nbytes)
 
     return a
 
 
+cdef ndarray _send_numpy_array_list_to_gpu(
+    list arrays, dtype, const vector.vector[Py_ssize_t]& shape, str order,
+        Py_ssize_t ndmin):
+    cdef ndarray a  # allocate it after pinned memory is secured
+    cdef size_t itemcount = internal.prod(shape)
+    cdef size_t nbytes = itemcount * get_dtype(dtype).itemsize
+
+    stream = stream_module.get_current_stream()
+    cdef pinned_memory.PinnedMemoryPointer mem = _alloc_pinned_memory(nbytes)
+    cdef size_t offset, length
+    if mem is not None:
+        # write concatenated arrays to the pinned memory directly
+        src_cpu = numpy.frombuffer(mem, dtype, itemcount).reshape(
+            shape, order=order)
+        numpy.concatenate([e[None] for e in arrays], 0, src_cpu)
+        a = ndarray(shape, dtype=dtype, order=order)
+        a.data.copy_from_host_async(ctypes.c_void_p(mem.ptr), nbytes)
+        pinned_memory._add_to_watch_list(stream.record(), mem)
+    else:
+        # fallback to numpy array and send it to GPU
+        a = _send_object_to_gpu(arrays, dtype, order, ndmin)
path: cupy/core/core.pyx
text: @user1:
Is this fallback meaningful?
Doesn't it cause OOM again in `_send_object_to_gpu`?

@author:
I'll fix to call `a.data.copy_from_host()` directly.

diff_hunk: @@ -128,6 +129,70 @@ def assert_array_list_equal(xlist, ylist, err_msg='', verbose=True):
             verbose=verbose)
 
 
+def _get_underlying_bytes(a):
+    if isinstance(a, cupy.ndarray):
+        # copy the array to host memory while preserving order
+        buf = ctypes.create_string_buffer(a.nbytes)
+        a.data.copy_to_host(ctypes.c_void_p(ctypes.addressof(buf)), a.nbytes)
+        arr = bytearray(a.nbytes)
+        arr[:] = buf.raw
+        return arr
+    else:
+        return ctypes.string_at(a.ctypes.data, a.nbytes)
+
+
+def assert_array_exactly_equal(x, y, err_msg='', verbose=True):
path: cupy/testing/array.py
text: @user1:
Does this function compare garbage bytes between elements?
Is there any point to do that? How about comparing strides instead?