Repository: chainer/chainer

Issue:
Parameter._grad stays on CPU after `to_gpu`
related to #6119. This only occurs after v6.0.0b1.

Code to reproduce
```hoge.py
>>> chainer.print_runtime_info()
Platform: Linux-4.4.0-135-generic-x86_64-with-Ubuntu-16.04-xenial
Chainer: 6.0.0b1
NumPy: 1.15.4
CuPy:
  CuPy Version          : 6.0.0b1
  CUDA Root             : /home/hashimoto/.local/cuda/cuda-9.2
  CUDA Build Version    : 9020
  CUDA Driver Version   : 10000
  CUDA Runtime Version  : 9020
  cuDNN Build Version   : 7401
  cuDNN Version         : 7401
  NCCL Build Version    : 2307
iDeep: Not Available
```
```fuga.py
from chainer import Parameter

p = Parameter(shape=(3, 3))
p.to_gpu()
print(type(p._grad))
```
```output.py
<class 'numpy.ndarray'>
```




Discussion:

Resolving patch:
diff --git a/chainer/variable.py b/chainer/variable.py
--- a/chainer/variable.py
+++ b/chainer/variable.py
@@ -640,6 +640,8 @@ def _set_chainerx_array(self, array, grad):
         assert array is None or isinstance(array, chainerx.ndarray)
         requires_grad = self._requires_grad
 
+        self._grad = None
+
         if (not requires_grad
                 and array is not None
                 and array.is_backprop_required()):
@@ -1161,6 +1163,8 @@ def _to_device(self, device, allow_unchaining):
             self._data = [new_arr]
             if grad_var is not None:
                 grad_var.to_device(device)
+                # _grad has been invalidated by grad_var.to_device().
+                self._grad = grad_var.array
 
         # ensure that the node tracks the device migration
         node = self._node


Test patch:
diff --git a/tests/chainer_tests/test_variable.py b/tests/chainer_tests/test_variable.py
--- a/tests/chainer_tests/test_variable.py
+++ b/tests/chainer_tests/test_variable.py
@@ -1478,96 +1478,59 @@ def test_update_rule_without_grad(self):
         assert update_rule.update.call_count == 1
 
 
+@testing.inject_backend_tests(
+    None,
+    [
+        {},
+        {'use_ideep': 'always'},
+        {'use_cuda': True, 'cuda_device': 0},
+        {'use_cuda': True, 'cuda_device': 1},
+        {'use_chainerx': True, 'chainerx_device': 'native:0'},
+        {'use_chainerx': True, 'chainerx_device': 'cuda:0'},
+        {'use_chainerx': True, 'chainerx_device': 'cuda:1'},
+    ])
 @testing.parameterize(
     {'x_shape': (10,)},
     {'x_shape': ()},
 )
 class TestParameterToDevice(unittest.TestCase):
 
-    def check_to_device(self, x, device_spec, expected_xp):
+    def check_to_device(self, x, device):
+        expected_xp = device.xp
         assert isinstance(x, chainer.Parameter)
-        x.to_device(device_spec)
+        x.to_device(device)
         assert x.xp is expected_xp
         assert x._has_chainerx_array is (expected_xp is chainerx)
 
-    def check_initializer(self, shape, device_spec, expected_xp):
-        x = chainer.Parameter(shape=shape)
-        self.check_to_device(x, device_spec, expected_xp)
+    def test_initializer_to_device(self, backend_config):
+        x = chainer.Parameter(shape=self.x_shape)
+        self.check_to_device(x, backend_config.device)
 
-    def check_initialize_by_scalar(self, shape, device_spec, expected_xp):
-        x = chainer.Parameter(2., shape)
-        self.check_to_device(x, device_spec, expected_xp)
+    def test_initialize_by_scalar_to_device(self, backend_config):
+        x = chainer.Parameter(2., self.x_shape)
+        self.check_to_device(x, backend_config.device)
 
-    def check_initialize_by_initializer(self, shape, device_spec, expected_xp):
-        x = chainer.Parameter(initializers.One(), shape)
-        self.check_to_device(x, device_spec, expected_xp)
+    def test_initialize_by_initializer_to_device(self, backend_config):
+        x = chainer.Parameter(initializers.One(), self.x_shape)
+        self.check_to_device(x, backend_config.device)
 
-    def check_initialize_by_none(self, shape, device_spec, expected_xp):
-        x = chainer.Parameter(None, shape)
-        self.check_to_device(x, device_spec, expected_xp)
+    def test_initialize_by_none_to_device(self, backend_config):
+        x = chainer.Parameter(None, self.x_shape)
+        self.check_to_device(x, backend_config.device)
 
-    def check_initialize_by_array(self, shape, device_spec, expected_xp):
-        data = np.random.uniform(-1, 1, shape).astype('f')
+    def test_initialize_by_array_to_device(self, backend_config):
+        data = np.random.uniform(-1, 1, self.x_shape).astype(np.float32)
         x = chainer.Parameter(data)
-        self.check_to_device(x, device_spec, expected_xp)
-
-    def test_initializer_to_device_numpy(self):
-        self.check_initializer(self.x_shape, np, np)
-
-    @attr.gpu
-    def test_initializer_to_device_cupy(self):
-        self.check_initializer(self.x_shape, (cuda.cupy, 0), cuda.cupy)
-
-    @attr.chainerx
-    def test_initializer_to_device_chainerx(self):
-        self.check_initializer(self.x_shape, 'native:0', chainerx)
-
-    def test_initialize_by_scalar_to_device_numpy(self):
-        self.check_initialize_by_scalar(self.x_shape, np, np)
-
-    @attr.gpu
-    def test_initialize_by_scalar_to_device_cupy(self):
-        self.check_initialize_by_scalar(
-            self.x_shape, (cuda.cupy, 0), cuda.cupy)
-
-    @attr.chainerx
-    def test_initialize_by_scalar_to_device_chainerx(self):
-        self.check_initialize_by_scalar(self.x_shape, 'native:0', chainerx)
-
-    def test_initialize_by_initializer_to_device_numpy(self):
-        self.check_initialize_by_initializer(self.x_shape, np, np)
-
-    @attr.gpu
-    def test_initialize_by_initializer_to_device_cupy(self):
-        self.check_initialize_by_initializer(
-            self.x_shape, (cuda.cupy, 0), cuda.cupy)
-
-    @attr.chainerx
-    def test_initialize_by_initializer_to_device_chainerx(self):
-        self.check_initialize_by_initializer(
-            self.x_shape, 'native:0', chainerx)
-
-    def test_initialize_by_none_to_device_numpy(self):
-        self.check_initialize_by_none(self.x_shape, np, np)
-
-    @attr.gpu
-    def test_initialize_by_none_to_device_cupy(self):
-        self.check_initialize_by_none(self.x_shape, (cuda.cupy, 0), cuda.cupy)
-
-    @attr.chainerx
-    def test_initialize_by_none_to_device_chainerx(self):
-        self.check_initialize_by_none(self.x_shape, 'native:0', chainerx)
-
-    def test_initialize_by_array_to_device_numpy(self):
-        self.check_initialize_by_array(self.x_shape, np, np)
-
-    @attr.gpu
-    def test_initialize_by_array_to_device_cupy(self):
-        self.check_initialize_by_array(self.x_shape, (cuda.cupy, 0), cuda.cupy)
-
-    @attr.chainerx
-    def test_initialize_by_array_to_device_chainerx(self):
-        self.check_initialize_by_array(self.x_shape, 'native:0', chainerx)
+        self.check_to_device(x, backend_config.device)
+
+    def test_internal_grad(self, backend_config):
+        device = backend_config.device
+        p = chainer.Parameter(shape=self.x_shape)
+        p.to_device(device)
+        if device.xp is chainerx:
+            assert p._grad is None
+        else:
+            assert isinstance(p._grad, device.supported_array_types)
 
 
 @testing.parameterize(
@@ -1705,6 +1668,33 @@ def test_initialize_by_array_from_chainerx_chainerx_cuda(self):
             self.x_shape, chainerx, cuda.cupy, 'cuda:0')
 
 
+@testing.inject_backend_tests(
+    None,
+    [
+        {},
+        {'use_ideep': 'always'},
+        {'use_cuda': True, 'cuda_device': 0},
+        {'use_cuda': True, 'cuda_device': 1},
+    ])
+class TestParameterToXpu(unittest.TestCase):
+
+    def _to_xpu(self, var, device):
+        if isinstance(device, backend.CpuDevice):
+            var.to_cpu()
+        elif isinstance(device, backend.GpuDevice):
+            var.to_gpu(device.device.id)
+        elif isinstance(device, backend.Intel64Device):
+            var.to_intel64()
+        else:
+            assert False
+
+    def test_internal_grad(self, backend_config):
+        device = backend_config.device
+        p = chainer.Parameter(shape=(2, 3))
+        self._to_xpu(p, device)
+        assert isinstance(p._grad, device.supported_array_types)
+
+
 class TestUninitializedParameter(unittest.TestCase):
 
     def setUp(self):