Repository: chainer/chainer

Issue:
Asynchronous Host-to-Device copy
Current CuPy does not have a way to allocate page-locked host memory to enable asynchronous HtoD copy. There are e.g. two options to support asynchronous HtoD copy:
1. Allocate page-locked host memory and copy the source buffer to it implicitly in the MemoryPointer.copy_from_host_async() method. It adds an overhead on copying, which might be problematic.
2. Provide a way to create a numpy array on page-locked host memory. I do not investigate the feasibility yet (maybe it can be done with using np.frombuffer and adding a destructor?).



Discussion:

Resolving patch:
diff --git a/chainer/cuda.py b/chainer/cuda.py
--- a/chainer/cuda.py
+++ b/chainer/cuda.py
@@ -96,6 +96,9 @@ class DummyDeviceType(object):
     This class is used to represent CPU device.
 
     """
+
+    id = -1
+
     def __int__(self):
         return -1
 
@@ -177,7 +180,8 @@ def to_gpu(array, device=None, stream=None):
     Args:
         array: Array to be sent to GPU.
         device: Device specifier.
-        stream (cupy.cuda.Stream): CUDA stream.
+        stream (cupy.cuda.Stream): CUDA stream. If not ``None``, the copy runs
+            asynchronously.
 
     Returns:
         cupy.ndarray: Array on GPU.
@@ -188,15 +192,33 @@ def to_gpu(array, device=None, stream=None):
 
     """
     check_cuda_available()
-    assert stream is None  # TODO(beam2d): FIX IT
     with get_device(device):
-        dev_id = int(get_device(array))
-        if dev_id != -1 and dev_id != cupy.cuda.device.get_device_id():
-            # Need to make a copy when an array is copied to another device
-            return cupy.array(array, copy=True)
-        else:
+        array_dev = get_device(array)
+        if array_dev.id == cupy.cuda.device.get_device_id():
+            return array
+
+        if stream is not None:
+            ret = cupy.empty_like(array)
+            if array_dev.id == -1:
+                # cpu to gpu
+                src = array.copy(order='C')
+                ret.set(src, stream)
+            else:
+                # gpu to gpu
+                with array_dev:
+                    src = array.copy()
+                ret.data.copy_from_device_async(src.data, src.nbytes, stream)
+
+            # to hold a reference until the end of the asynchronous memcpy
+            stream.add_callback(lambda *x: None, (src, ret))
+            return ret
+
+        if array_dev.id == -1:
             return cupy.asarray(array)
 
+        # Need to make a copy when an array is copied to another device
+        return cupy.array(array, copy=True)
+
 
 def to_cpu(array, stream=None):
     """Copies the given GPU array to host CPU.
diff --git a/cupy/cuda/stream.py b/cupy/cuda/stream.py
--- a/cupy/cuda/stream.py
+++ b/cupy/cuda/stream.py
@@ -128,7 +128,9 @@ def add_callback(self, callback, arg):
             arg (object): Argument to the callback.
 
         """
-        runtime.streamAddCallback(self.ptr, callback, arg)
+        def f(stream, status, dummy):
+            callback(self, status, arg)
+        runtime.streamAddCallback(self.ptr, f, 0)
 
     def record(self, event=None):
         """Records an event on the stream.


Test patch:
diff --git a/tests/chainer_tests/test_cuda.py b/tests/chainer_tests/test_cuda.py
--- a/tests/chainer_tests/test_cuda.py
+++ b/tests/chainer_tests/test_cuda.py
@@ -111,4 +111,98 @@ def test_size(self):
         self.assertEqual(size, cuda.get_max_workspace_size())
 
 
+@testing.parameterize(
+    {'c_contiguous': True},
+    {'c_contiguous': False},
+)
+class TestToGPU(unittest.TestCase):
+
+    def setUp(self):
+        self.x = numpy.random.uniform(-1, 1, (2, 3))
+        if not self.c_contiguous:
+            self.x = self.x.T
+
+    @attr.gpu
+    def test_numpy_array(self):
+        y = cuda.to_gpu(self.x)
+        self.assertIsInstance(y, cuda.ndarray)
+        cuda.cupy.testing.assert_array_equal(self.x, y)
+
+    @attr.gpu
+    def test_cupy_array1(self):
+        x = cuda.to_gpu(self.x)
+        y = cuda.to_gpu(x)
+        self.assertIsInstance(y, cuda.ndarray)
+        self.assertIs(x, y)  # Do not copy
+
+    @attr.multi_gpu(2)
+    def test_cupy_array2(self):
+        x = cuda.to_gpu(self.x, device=0)
+        with x.device:
+            if not self.c_contiguous:
+                x = cuda.cupy.asfortranarray(x)
+        y = cuda.to_gpu(x, device=1)
+        self.assertIsInstance(y, cuda.ndarray)
+        self.assertEqual(int(y.device), 1)
+
+    @attr.gpu
+    def test_numpy_array_async(self):
+        y = cuda.to_gpu(self.x, stream=cuda.Stream(null=True))
+        self.assertIsInstance(y, cuda.ndarray)
+        cuda.cupy.testing.assert_array_equal(self.x, y)
+
+    @attr.multi_gpu(2)
+    def test_numpy_array_async2(self):
+        y = cuda.to_gpu(self.x, device=1, stream=cuda.Stream(null=True))
+        self.assertIsInstance(y, cuda.ndarray)
+        cuda.cupy.testing.assert_array_equal(self.x, y)
+        self.assertEqual(int(y.device), 1)
+
+    @attr.multi_gpu(2)
+    def test_numpy_array_async3(self):
+        with cuda.Device(1):
+            y = cuda.to_gpu(self.x, stream=cuda.Stream(null=True))
+        self.assertIsInstance(y, cuda.ndarray)
+        cuda.cupy.testing.assert_array_equal(self.x, y)
+        self.assertEqual(int(y.device), 1)
+
+    @attr.gpu
+    def test_cupy_array_async1(self):
+        x = cuda.to_gpu(self.x)
+        if not self.c_contiguous:
+            x = cuda.cupy.asfortranarray(x)
+        y = cuda.to_gpu(x, stream=cuda.Stream())
+        self.assertIsInstance(y, cuda.ndarray)
+        self.assertIs(x, y)  # Do not copy
+        cuda.cupy.testing.assert_array_equal(x, y)
+
+    @attr.multi_gpu(2)
+    def test_cupy_array_async2(self):
+        x = cuda.to_gpu(self.x, device=0)
+        with x.device:
+            if not self.c_contiguous:
+                x = cuda.cupy.asfortranarray(x)
+        y = cuda.to_gpu(x, device=1, stream=cuda.Stream(null=True))
+        self.assertIsInstance(y, cuda.ndarray)
+        self.assertIsNot(x, y)  # Do copy
+        cuda.cupy.testing.assert_array_equal(x, y)
+
+    @attr.multi_gpu(2)
+    def test_cupy_array_async3(self):
+        with cuda.Device(0):
+            x = cuda.to_gpu(self.x)
+            if not self.c_contiguous:
+                x = cuda.cupy.asfortranarray(x)
+        with cuda.Device(1):
+            y = cuda.to_gpu(x, stream=cuda.Stream(null=True))
+        self.assertIsInstance(y, cuda.ndarray)
+        self.assertIsNot(x, y)  # Do copy
+        cuda.cupy.testing.assert_array_equal(x, y)
+
+    def test_variable_cpu(self):
+        x = chainer.Variable(self.x)
+        with self.assertRaises(TypeError):
+            cuda.to_cpu(x)
+
+
 testing.run_module(__name__, __file__)