Repository: chainer/chainer

Issue:
Link.to_gpu of Link.copy is inconsistent if there is an uninitialized Variable
For data-parallel training the following code works because `Link.to_gpu` makes in-place copies of parameters on a GPU.
```
    model_0 = L.Classifier(MLP())
    model_0.predictor(np.zeros((1, 784), dtype='f'))
    model_1 = model_0.copy()
    model_0.to_gpu(0)
    model_1.to_gpu(1)
```
The issue is that the second line of the code is required.

cf. The following code works, too.
```
    model_0 = L.Classifier(MLP())
    model_1 = copy.deepcopy(model_0)
    model_0.to_gpu(0)
    model_1.to_gpu(1)
```

To reproduce the issue:
```
import chainer
import chainer.links as L
import cupy
l0 = L.Linear(None, 10)
l1 = l0.copy()
l0.to_gpu(0)
l1.to_gpu(1)
print(l1.W.data)  # =>  None
print(l1.b.data.device)  # =>  <CUDA Device 1>
l0(cupy.random.randn(3, 4).astype('f'))
print(l1.W.data.device)  # =>  <CUDA Device 0>  (!!)
print(l1.b.data.device)  # =>  <CUDA Device 1>
print(l0.W.data is l1.W.data)  # =>  True  (!!)
```

Related: #1953


Discussion:

Resolving patch:
diff --git a/chainer/variable.py b/chainer/variable.py
--- a/chainer/variable.py
+++ b/chainer/variable.py
@@ -709,6 +709,7 @@ def to_gpu(self, device=None):
         if self.data is None:
             self._initial_device = (cuda.Device().id
                                     if device is None else device)
+            self._data = [None]  # Renew placeholder to break sharing
         else:
             self._data = [cuda.to_gpu(self.data, device)]
             if self._grad_var is not None:


Test patch:
diff --git a/tests/chainer_tests/test_link.py b/tests/chainer_tests/test_link.py
--- a/tests/chainer_tests/test_link.py
+++ b/tests/chainer_tests/test_link.py
@@ -154,6 +154,47 @@ def test_copy(self):
         self.assertIs(link.p, self.link.p)
         self.assertIs(link.name, None)
 
+    @attr.gpu
+    def test_copy_and_to_gpu_init(self):
+        cupy = cuda.cupy
+        l0 = self.link
+        l1 = l0.copy()
+        self.assertIs(l0.x.data, l1.x.data)
+        l1.to_gpu()
+        self.assertIsNot(l0.x.data, l1.x.data)
+        self.assertIsInstance(l0.x.data, numpy.ndarray)
+        self.assertIsInstance(l1.x.data, cupy.ndarray)
+
+    @attr.gpu
+    def test_copy_and_to_gpu_uninit(self):
+        cupy = cuda.cupy
+        l0 = self.link
+        l1 = l0.copy()
+        self.assertIsNone(l0.u.data)
+        self.assertIsNone(l1.u.data)
+        l1.to_gpu()
+        l1.u.initialize((2, 3))
+        self.assertIsNone(l0.u.data)
+        self.assertIsInstance(l1.u.data, cupy.ndarray)
+
+    @attr.multi_gpu(2)
+    def test_copy_and_to_gpu_uninit_multi_gpu(self):
+        cupy = cuda.cupy
+        l0 = self.link
+        l1 = l0.copy()
+        l2 = l0.copy()
+        self.assertIsNone(l0.u.data)
+        self.assertIsNone(l1.u.data)
+        self.assertIsNone(l2.u.data)
+        l1.to_gpu()
+        l1.u.initialize((2, 3))
+        l2.to_gpu()
+        l2.u.initialize((2, 3))
+        self.assertIsNone(l0.u.data)
+        self.assertIsInstance(l1.u.data, cupy.ndarray)
+        self.assertIsInstance(l2.u.data, cupy.ndarray)
+        self.assertNotEqual(l1.u.data.data, l2.u.data.data)
+
     def _check_deepcopy(self, link):
         self.assertIsInstance(link._params, set)
         self.assertIsInstance(link._persistent, set)