Repository: chainer/chainer

Issue:
Mix CPU mode and GPU mode in one network
I want to use CPU mode for some functions and GPU mode for others in one network.
When I use a large number of vocabulary in EmbedID about >=1,000,000 words, it consumes large size of memory in GPU. In this situation, I need to use EmbedID in CPU, and to use other functions like LSTMs in GPU.



Discussion:

Resolving patch:
diff --git a/chainer/functions/array/copy.py b/chainer/functions/array/copy.py
--- a/chainer/functions/array/copy.py
+++ b/chainer/functions/array/copy.py
@@ -16,24 +16,45 @@ def check_type_forward(self, in_types):
         )
 
     def forward_cpu(self, x):
-        return x[0].copy(),
+        if self.out_device == -1:
+            return x[0].copy(),
+        else:
+            return cuda.to_gpu(x[0], device=self.out_device),
 
     def forward_gpu(self, x):
-        return cuda.copy(x[0], out_device=self.out_device),
+        if self.out_device == -1:
+            return cuda.to_cpu(x[0]),
+        else:
+            return cuda.copy(x[0], out_device=self.out_device),
+
+    def backward(self, inputs, grad_outputs):
+        # In this function, `grad_outputs` contains cuda arrays even when
+        # `inputs` only contains numpy arrays.
+        if isinstance(inputs[0], cuda.ndarray):
+            return self.backward_gpu(inputs, grad_outputs)
+        else:
+            return self.backward_cpu(inputs, grad_outputs)
 
     def backward_cpu(self, x, gy):
-        return gy[0].copy(),
+        if self.out_device == -1:
+            return gy[0].copy(),
+        else:
+            return cuda.to_cpu(gy[0]),
 
     def backward_gpu(self, x, gy):
-        return cuda.copy(gy[0], out_device=cuda.get_device(x[0])),
+        if self.out_device == -1:
+            return cuda.to_gpu(gy[0], device=cuda.get_device(x[0])),
+        else:
+            return cuda.copy(gy[0], out_device=cuda.get_device(x[0])),
 
 
 def copy(x, dst):
     """Copies the input variable onto the specified device.
 
     This function copies the array of input variable onto the device specified
-    by ``dst`` if the original array is on GPU, and otherwise just copies the
-    array within host memory.
+    by ``dst``. When ``dst == -1``, it copies the array onto the host memory.
+    This function supports copies from host to device, from device to device
+    and from device to host.
 
     Args:
         x (~chainer.Variable): Variable to be copied.


Test patch:
diff --git a/tests/chainer_tests/functions_tests/array_tests/test_copy.py b/tests/chainer_tests/functions_tests/array_tests/test_copy.py
--- a/tests/chainer_tests/functions_tests/array_tests/test_copy.py
+++ b/tests/chainer_tests/functions_tests/array_tests/test_copy.py
@@ -63,6 +63,26 @@ def test_check_backward_gpu(self):
         device_id = cuda.Device().id
         self.check_forward(device_id, device_id)
 
+    @attr.gpu
+    def test_forward_cpu_to_gpu(self):
+        device_id = cuda.Device().id
+        self.check_forward(-1, device_id)
+
+    @attr.gpu
+    def test_backward_cpu_to_gpu(self):
+        device_id = cuda.Device().id
+        self.check_backward(-1, device_id)
+
+    @attr.gpu
+    def test_forward_gpu_to_cpu(self):
+        device_id = cuda.Device().id
+        self.check_forward(device_id, -1)
+
+    @attr.gpu
+    def test_backward_gpu_to_cpu(self):
+        device_id = cuda.Device().id
+        self.check_backward(device_id, -1)
+
     @attr.multi_gpu(2)
     def test_forward_multigpu(self):
         self.check_forward(0, 1)