Repository: chainer/chainer

Issue:
Should `copyparams` copy persistent values as well?
Currently, `{Link, Chain}.copyparams` does not care persistent values and it only copies parameters. Due to this behavior, for example, *every* model using batch normalization cannot be correctly copied using this method.

See #4902 for an example of errors due to this problem.

Do we have situations where we want to copy parameter values but we don't want to copy persistent values? I propose to change the behavior of `copyparams` to also copy persistent values.



Discussion:

Resolving patch:
diff --git a/chainer/backends/cuda.py b/chainer/backends/cuda.py
--- a/chainer/backends/cuda.py
+++ b/chainer/backends/cuda.py
@@ -424,6 +424,35 @@ def copy(array, out=None, out_device=None, stream=None):
     return out
 
 
+def copyto(dst, src):
+    """Copies the elements of an ndarray to those of another one.
+
+    This function can copy the CPU/GPU arrays to the destination arrays on
+    another device.
+
+    Args:
+        dst (numpy.ndarray or cupy.ndarray): Destination array.
+        src (numpy.ndarray or cupy.ndarray): Source array.
+
+    """
+    if isinstance(dst, numpy.ndarray):
+        numpy.copyto(dst, to_cpu(src))
+    elif isinstance(dst, ndarray):
+        if isinstance(src, numpy.ndarray):
+            if dst.flags.c_contiguous or dst.flags.f_contiguous:
+                dst.set(src)
+            else:
+                cupy.copyto(dst, to_gpu(src, device=dst.device))
+        elif isinstance(src, ndarray):
+            cupy.copyto(dst, src)
+        else:
+            raise TypeError('cannot copy from non-array object of type {}'
+                            .format(type(src)))
+    else:
+        raise TypeError('cannot copy to non-array object of type {}'.format(
+            type(dst)))
+
+
 # ------------------------------------------------------------------------------
 # Function result memoization
 # ------------------------------------------------------------------------------
diff --git a/chainer/link.py b/chainer/link.py
--- a/chainer/link.py
+++ b/chainer/link.py
@@ -508,21 +508,38 @@ def children(self):
         if 0:
             yield
 
-    def copyparams(self, link):
+    def copyparams(self, link, copy_persistent=True):
         """Copies all parameters from given link.
 
         This method copies data arrays of all parameters in the hierarchy. The
         copy is even done across the host and devices. Note that this method
         does not copy the gradient arrays.
 
+        *From v5.0.0:* this method also copies the persistent values (e.g. the
+        moving statistics of :class:`~chainer.links.BatchNormalization`). If
+        the persistent value is an ndarray, the elements are copied. Otherwise,
+        it is copied using :func:`copy.deepcopy`. The old behavior (not copying
+        persistent values) can be reproduced with ``copy_persistent=False``.
+
         Args:
             link (Link): Source link object.
+            copy_persistent (bool): If ``True``, persistent values are also
+                copied. ``True`` by default.
 
         """
         src = link.__dict__
         dst = self.__dict__
         for name in self._params:
             dst[name].copydata(src[name])
+        if copy_persistent:
+            array_types = chainer.get_array_types()
+            for name in self._persistent:
+                d = dst[name]
+                s = src[name]
+                if isinstance(d, array_types) and isinstance(s, array_types):
+                    cuda.copyto(d, s)
+                else:
+                    dst[name] = copy.deepcopy(s)
 
     def cleargrads(self):
         """Clears all gradient arrays.
@@ -933,12 +950,12 @@ def children(self):
         for name in self._children:
             yield d[name]
 
-    def copyparams(self, link):
-        super(Chain, self).copyparams(link)
+    def copyparams(self, link, copy_persistent=True):
+        super(Chain, self).copyparams(link, copy_persistent)
         src = link.__dict__
         dst = self.__dict__
         for name in self._children:
-            dst[name].copyparams(src[name])
+            dst[name].copyparams(src[name], copy_persistent)
 
     def addgrads(self, link):
         super(Chain, self).addgrads(link)
@@ -1116,10 +1133,10 @@ def children(self):
         for child in self._children:
             yield child
 
-    def copyparams(self, link):
-        super(ChainList, self).copyparams(link)
+    def copyparams(self, link, copy_persistent=True):
+        super(ChainList, self).copyparams(link, copy_persistent)
         for idx, child in enumerate(self._children):
-            child.copyparams(link[idx])
+            child.copyparams(link[idx], copy_persistent)
 
     def addgrads(self, link):
         super(ChainList, self).addgrads(link)
diff --git a/chainer/variable.py b/chainer/variable.py
--- a/chainer/variable.py
+++ b/chainer/variable.py
@@ -850,14 +850,7 @@ def copydata(self, var):
         elif dst is None:
             self.initialize(src.shape)
             dst = self.data
-        src_xp = cuda.get_array_module(src)
-        dst_xp = cuda.get_array_module(dst)
-        if dst_xp is src_xp:
-            dst_xp.copyto(dst, src)
-        elif dst_xp is numpy:
-            dst_xp.copyto(dst, src.get())
-        else:
-            dst.set(src)
+        cuda.copyto(dst, src)
 
     def addgrad(self, var):
         """Accumulates the gradient array from given source variable.


Test patch:
diff --git a/tests/chainer_tests/test_cuda.py b/tests/chainer_tests/test_cuda.py
--- a/tests/chainer_tests/test_cuda.py
+++ b/tests/chainer_tests/test_cuda.py
@@ -301,6 +301,56 @@ def test_numpy_scalar(self):
         assert y == x
 
 
+class TestCopyTo(unittest.TestCase):
+
+    def test_cpu_to_cpu(self):
+        src = numpy.arange(1, 5, dtype=numpy.float32)
+        dst = numpy.zeros_like(src)
+        cuda.copyto(dst, src)
+        numpy.testing.assert_array_equal(dst, src)
+
+    @attr.gpu
+    def test_cpu_to_gpu(self):
+        src = numpy.arange(1, 5, dtype=numpy.float32)
+        dst = cuda.cupy.zeros_like(src)
+        cuda.copyto(dst, src)
+        cuda.cupy.testing.assert_array_equal(dst, src)
+
+    @attr.gpu
+    def test_gpu_to_cpu(self):
+        src = cuda.cupy.arange(1, 5, dtype=numpy.float32)
+        dst = numpy.zeros_like(src.get())
+        cuda.copyto(dst, src)
+        cuda.cupy.testing.assert_array_equal(dst, src)
+
+    @attr.gpu
+    def test_gpu_to_gpu(self):
+        src = cuda.cupy.arange(1, 5, dtype=numpy.float32)
+        dst = cuda.cupy.zeros_like(src)
+        cuda.copyto(dst, src)
+        cuda.cupy.testing.assert_array_equal(dst, src)
+
+    @attr.multi_gpu(2)
+    def test_gpu_to_another_gpu(self):
+        src = cuda.cupy.arange(1, 5, dtype=numpy.float32)
+        with cuda.get_device_from_id(1):
+            dst = cuda.cupy.zeros_like(src)
+        cuda.copyto(dst, src)
+        cuda.cupy.testing.assert_array_equal(dst, src)
+
+    def test_fail_on_invalid_src(self):
+        src = None
+        dst = numpy.zeros(1)
+        with self.assertRaises(TypeError):
+            cuda.copyto(dst, src)
+
+    def test_fail_on_invalid_dst(self):
+        src = numpy.zeros(1)
+        dst = None
+        with self.assertRaises(TypeError):
+            cuda.copyto(dst, src)
+
+
 @attr.cudnn
 class TestWorkspace(unittest.TestCase):
 
diff --git a/tests/chainer_tests/test_link.py b/tests/chainer_tests/test_link.py
--- a/tests/chainer_tests/test_link.py
+++ b/tests/chainer_tests/test_link.py
@@ -361,7 +361,7 @@ def test_namedlinks(self):
         self.assertEqual(pl[0][0], '/')
         self.assertIs(pl[0][1], self.link)
 
-    def test_copyparams(self):
+    def _setup_test_copyparams(self):
         self.link.x.grad.fill(0)
         self.link.y.grad.fill(1)
         self.link.u.initialize((2, 3))
@@ -386,8 +386,12 @@ def test_copyparams(self):
         l.u.grad.fill(7)
         l.v.data.fill(8)
         l.v.grad.fill(9)
+        l.add_persistent('p', numpy.full_like(self.link.p, 10))
 
-        self.link.copyparams(l)
+        return l, (gx, gy, gu)
+
+    def _check_copyparams(self, l, gs):
+        gx, gy, gu = gs
         numpy.testing.assert_array_equal(self.link.x.data, l.x.data)
         numpy.testing.assert_array_equal(self.link.x.grad, gx)
         numpy.testing.assert_array_equal(self.link.y.data, l.y.data)
@@ -397,6 +401,22 @@ def test_copyparams(self):
         numpy.testing.assert_array_equal(self.link.v.data, l.v.data)
         numpy.testing.assert_array_equal(self.link.v.grad, None)
 
+    def test_copyparams(self):
+        l, gs = self._setup_test_copyparams()
+        self.link.copyparams(l)
+        self._check_copyparams(l, gs)
+        numpy.testing.assert_array_equal(self.link.p, l.p)
+
+    def test_copyparams_no_copy_persistent(self):
+        orig_p = self.link.p.copy()
+
+        l, gs = self._setup_test_copyparams()
+        numpy.testing.assert_array_equal(False, orig_p == l.p)
+        self.link.copyparams(l, copy_persistent=False)
+
+        self._check_copyparams(l, gs)
+        numpy.testing.assert_array_equal(self.link.p, orig_p)
+
     def test_cleargrads(self):
         self.link.cleargrads()
         self.assertIsNone(self.link.x.grad)