Repository: chainer/chainer

Issue:
Use `...` instead of `:`
### Code to reproduce
```python
import chainer


class MyLink(chainer.Link):
    def __init__(self, v):
        super().__init__()
        with self.init_scope():
            self.v = v


l1 = MyLink(chainer.Parameter(1, shape=()))
chainer.serializers.save_npz('tmp.npz', l1)

l2 = MyLink(chainer.Parameter(2, shape=None))
#(1.) chainer.serializers.load_npz('tmp.npz', l2)

opt = chainer.optimizers.Adam(weight_decay_rate=1e-4)
opt.setup(l1)
l1.v.grad = l1.v.array.copy()
#(2.) opt.update()
```

### Error messages, stack traces, or logs
1. Deserializing a link with ()-shaped uninitialized parameter fails:
   ```
   Traceback (most recent call last):
     File "new_issue.py", line 15, in <module>
       chainer.serializers.load_npz('tmp.npz', l2)
     File "/Users/tos/GitHub/chainer/chainer/serializers/npz.py", line 242, in load_npz
       d.load(obj)
     File "/Users/tos/GitHub/chainer/chainer/serializer.py", line 83, in load
       obj.serialize(self)
     File "/Users/tos/GitHub/chainer/chainer/link.py", line 661, in serialize
       param.data[:] = param.device.send(data)
   IndexError: too many indices for array
   ```

2. AdamW fails:
   ```
   Traceback (most recent call last):
     File "new_issue.py", line 20, in <module>
       opt.update()
     File "/Users/tos/GitHub/chainer/chainer/optimizer.py", line 815, in update
       param.update()
     File "/Users/tos/GitHub/chainer/chainer/variable.py", line 1898, in update
       self.update_rule.update(self)
     File "/Users/tos/GitHub/chainer/chainer/optimizer.py", line 223, in update
       self.update_core(param)
     File "/Users/tos/GitHub/chainer/chainer/optimizer.py", line 242, in update_core
       self.update_core_cpu(param)
     File "/Users/tos/GitHub/chainer/chainer/optimizers/adam.py", line 220, in update_core_cpu
       param.data, 1.0 - hp.weight_decay_rate, -hp.eta, step * m)
     File "/Users/tos/GitHub/chainer/chainer/optimizers/adam.py", line 74, in _inplace_axpby
       x[:] = a * x + b * y
   IndexError: too many indices for array
   ```


Discussion:

Resolving patch:
diff --git a/chainer/link.py b/chainer/link.py
--- a/chainer/link.py
+++ b/chainer/link.py
@@ -658,7 +658,8 @@ def serialize(self, serializer):
             if param.data is None and data is not None:
                 # Initialize the parameter here
                 param.initialize(data.shape)
-                param.data[:] = param.device.send(data)
+                with chainer.using_device(param.device):
+                    param.data[...] = param.device.send(data)
         for name in self._persistent:
             d[name] = serializer(name, d[name])
 
diff --git a/chainer/optimizers/adam.py b/chainer/optimizers/adam.py
--- a/chainer/optimizers/adam.py
+++ b/chainer/optimizers/adam.py
@@ -71,7 +71,7 @@ def _inplace_axpby(x, a, b, y):
         if a == 1:
             x += b * y
         else:
-            x[:] = a * x + b * y
+            x[...] = a * x + b * y
 
 
 class AdamRule(optimizer.UpdateRule):


Test patch:
diff --git a/tests/chainer_tests/optimizers_tests/test_optimizers.py b/tests/chainer_tests/optimizers_tests/test_optimizers.py
--- a/tests/chainer_tests/optimizers_tests/test_optimizers.py
+++ b/tests/chainer_tests/optimizers_tests/test_optimizers.py
@@ -5,27 +5,98 @@
 import numpy as np
 
 import chainer
+from chainer import functions as F
 from chainer import optimizers
 from chainer import testing
 
 
+_all_optimizers = [
+    'AdaDelta',
+    'AdaGrad',
+    'Adam',
+    'AdamW',
+    'AMSGrad',
+    'AdaBound',
+    'AMSBound',
+    'CorrectedMomentumSGD',
+    'MomentumSGD',
+    'MSVAG',
+    'NesterovAG',
+    'RMSprop',
+    'RMSpropGraves',
+    'SGD',
+    'SMORMS3',
+]
+
+
 _parameterize_optimizers = testing.parameterize(*testing.product({
-    'optimizer_impl': [
-        optimizers.AdaDelta,
-        optimizers.AdaGrad,
-        optimizers.Adam,
-        optimizers.CorrectedMomentumSGD,
-        optimizers.MomentumSGD,
-        optimizers.MSVAG,
-        optimizers.NesterovAG,
-        optimizers.RMSprop,
-        optimizers.RMSpropGraves,
-        optimizers.SGD,
-        optimizers.SMORMS3,
-    ]
+    'optimizer_impl': [getattr(chainer.optimizers, o) for o in _all_optimizers]
 }))
 
 
+class SimpleChain(chainer.Chain):
+
+    def __init__(self, shape=()):
+        super(SimpleChain, self).__init__()
+        w_np = np.asarray(np.random.randn(*shape)).astype(np.float32)
+        with self.init_scope():
+            self.w = chainer.Parameter(w_np, name='w')
+
+    def __call__(self, x):
+        return F.sum((x - self.w) ** 2)
+
+
+@testing.backend.inject_backend_tests(
+    None,
+    [
+        # CPU
+        {},
+        # Intel
+        {'use_ideep': True},
+        # CUDA
+        {'use_cuda': True, 'cuda_device': 0},
+        {'use_cuda': True, 'cuda_device': 1},
+        # ChainerX
+        {'use_chainerx': True, 'chainerx_device': 'native:0'},
+        {'use_chainerx': True, 'chainerx_device': 'cuda:0'},
+        {'use_chainerx': True, 'chainerx_device': 'cuda:1'},
+    ]
+)
+@testing.parameterize(*(
+    # Optimizers constructed with default arguments
+    [
+        {
+            'optimizer': o,
+            'kwargs': {}
+        }
+        for o in _all_optimizers]
+    # https://chainer/chainer/issues/7424
+    + [
+        {
+            'optimizer': 'Adam',
+            'kwargs': {'weight_decay_rate': 0.5},
+        }]
+))
+@testing.parameterize(*testing.product(
+    {'shape': [(2, 3), (), (1, 0, 2)]}
+))
+class TestOptimizer(unittest.TestCase):
+
+    def test_optimizer(self, backend_config):
+        device = backend_config.device
+        target = SimpleChain(self.shape)
+        target.to_device(device)
+        optimizer_cls = getattr(chainer.optimizers, self.optimizer)
+        optimizer = optimizer_cls(**self.kwargs)
+        optimizer.setup(target)
+
+        x_np = np.asarray(np.random.randn(*self.shape)).astype(np.float32)
+        x = chainer.Variable(device.send(x_np))
+
+        # Just ensures no error occurs. No numerical check is performed.
+        optimizer.update(target, x)
+
+
 @_parameterize_optimizers
 class TestOptimizerHyperparameter(unittest.TestCase):
 
@@ -42,6 +113,18 @@ def get_hyperparam(self, name):
         return getattr(self.target.w.update_rule.hyperparam, name)
 
     def test_hyperparams(self):
+        # TODO(niboshi): The following optimizers do not pass this test
+        # because their __init__ do not accept some hyperparameters.
+        # The test should be fixed.
+        if self.optimizer_impl in (
+                chainer.optimizers.AdamW,
+                chainer.optimizers.AMSGrad,
+                chainer.optimizers.AdaBound,
+                chainer.optimizers.AMSBound,
+        ):
+            raise unittest.SkipTest(
+                'The optimizer is incompatible with this test')
+
         self.create()
         default = self.optimizer.hyperparam.get_dict()
         for name, default_value in six.iteritems(default):
@@ -66,17 +149,6 @@ def __call__(self, rule, param):
         self.value = np.copy(p)
 
 
-class SimpleChain(chainer.Chain):
-
-    def __init__(self):
-        super(SimpleChain, self).__init__()
-        with self.init_scope():
-            self.w = chainer.Parameter(42, (), 'w')
-
-    def __call__(self, x):
-        return (x - self.w) ** 2
-
-
 @_parameterize_optimizers
 class TestOptimizerHooks(unittest.TestCase):
 
diff --git a/tests/chainer_tests/test_link.py b/tests/chainer_tests/test_link.py
--- a/tests/chainer_tests/test_link.py
+++ b/tests/chainer_tests/test_link.py
@@ -29,6 +29,44 @@ def _assert_arrays_equal(array, expected_array):
     assert (array == expected_array).all()
 
 
+def _shaped_random(shape, dtype):
+    if isinstance(shape, int):
+        shape = (shape,)
+    return numpy.asarray(numpy.random.randn(*shape)).astype(dtype)
+
+
+_inject_backend_tests = testing.backend.inject_backend_tests(
+    None,
+    [
+        # NumPy
+        {},
+        # iDeep
+        {'use_ideep': 'always'},
+        # CuPy
+        {'use_cuda': True, 'cuda_device': 0},
+        {'use_cuda': True, 'cuda_device': 1},
+        # ChainerX
+        {'use_chainerx': True, 'chainerx_device': 'native:0'},
+        {'use_chainerx': True, 'chainerx_device': 'cuda:0'},
+        {'use_chainerx': True, 'chainerx_device': 'cuda:1'},
+    ])
+
+
+_inject_backend_tests_no_intel64 = testing.backend.inject_backend_tests(
+    None,
+    [
+        # NumPy
+        {},
+        # CuPy
+        {'use_cuda': True, 'cuda_device': 0},
+        {'use_cuda': True, 'cuda_device': 1},
+        # ChainerX
+        {'use_chainerx': True, 'chainerx_device': 'native:0'},
+        {'use_chainerx': True, 'chainerx_device': 'cuda:0'},
+        {'use_chainerx': True, 'chainerx_device': 'cuda:1'},
+    ])
+
+
 class LinkTestBase(object):
 
     def setUp(self):
@@ -562,47 +600,6 @@ def test_addgrads(self):
         with pytest.raises(RuntimeError):
             v.grad
 
-    def test_serialize(self):
-        serializer = mock.MagicMock(return_value=3)
-        l = chainer.Link()
-        with l.init_scope():
-            l.x = chainer.Parameter(shape=(2, 3))
-            l.y = chainer.Parameter(shape=2)
-        l.add_persistent('z', 1)
-        l.serialize(serializer)
-        self.assertEqual(serializer.call_count, 3)
-        serializer.assert_any_call('x', l.x.data)
-        serializer.assert_any_call('y', l.y.data)
-        serializer.assert_any_call('z', 1)
-        self.assertEqual(l.z, 3)
-
-    def test_serialize_param_shape_placeholder(self):
-        serializer = mock.MagicMock(return_value=3)
-        l = chainer.Link()
-        with l.init_scope():
-            l.y = chainer.Parameter(shape=2)
-            l.x = chainer.Parameter()
-        l.x.initialize((2, 3))
-        l.add_persistent('z', 1)
-        l.serialize(serializer)
-        self.assertEqual(serializer.call_count, 3)
-        serializer.assert_any_call('x', l.x.data)
-        serializer.assert_any_call('y', l.y.data)
-        serializer.assert_any_call('z', 1)
-        self.assertEqual(l.z, 3)
-
-    def test_serialize_deserialize_to_uninitialized_param(self):
-        ret = numpy.random.rand(2, 3).astype('f')
-        serializer = mock.MagicMock(return_value=ret)
-        l = chainer.Link()
-        with l.init_scope():
-            l.x = chainer.Parameter()
-        l.serialize(serializer)
-        self.assertEqual(serializer.call_count, 1)
-        serializer.assert_any_call('x', None)
-        self.assertIsInstance(l.x.data, numpy.ndarray)
-        numpy.testing.assert_array_equal(l.x.data, ret)
-
     def test_enable_update(self):
         self.link.enable_update()
         self.assertTrue(self.link.x.update_rule.enabled)
@@ -635,19 +632,217 @@ def test_count_params(self):
         assert not w
 
 
-@testing.backend.inject_backend_tests(
-    None,
-    [
-        # NumPy
-        {},
-        # CuPy
-        {'use_cuda': True, 'cuda_device': 0},
-        {'use_cuda': True, 'cuda_device': 1},
-        # ChainerX
-        {'use_chainerx': True, 'chainerx_device': 'native:0'},
-        {'use_chainerx': True, 'chainerx_device': 'cuda:0'},
-        {'use_chainerx': True, 'chainerx_device': 'cuda:1'},
-    ])
+@_inject_backend_tests
+@testing.parameterize_pytest(
+    'shape_x,shape_y', [
+        ((2, 3), 2),
+        ((2, 3), ()),
+        ((2, 3), (1, 0, 3)),
+    ]
+)
+class TestLinkSerializeDeserialize(unittest.TestCase):
+
+    def test_serialize(self, backend_config):
+        call_record = []
+
+        def serializer(key, value):
+            call_record.append((key, value))
+            return value
+
+        l = chainer.Link()
+        with l.init_scope():
+            l.x = chainer.Parameter(shape=self.shape_x)
+            l.y = chainer.Parameter(shape=self.shape_y)
+        l.add_persistent('z', 1)
+        l.to_device(backend_config.device)
+
+        old_x_data = l.x.array
+        old_y_data = l.y.array
+        old_z = l.z
+
+        l.serialize(serializer)
+
+        # Link data are not modified
+        self.assertIs(l.x.array, old_x_data)
+        self.assertIs(l.y.array, old_y_data)
+        self.assertEqual(l.z, old_z)
+
+        # Check inputs to the serializer
+        self.assertEqual(len(call_record), 3)
+        call_record = sorted(call_record)
+        self.assertEqual(call_record[0][0], 'x')
+        self.assertIs(call_record[0][1], l.x.array)
+        self.assertEqual(call_record[1][0], 'y')
+        self.assertIs(call_record[1][1], l.y.array)
+        self.assertEqual(call_record[2][0], 'z')
+        self.assertEqual(call_record[2][1], old_z)
+
+    def test_deserialize(self, backend_config):
+        call_record = []
+
+        state = {
+            'x': _shaped_random(self.shape_x, 'float32'),
+            'y': _shaped_random(self.shape_y, 'float32'),
+            'z': numpy.random.randn(),
+        }
+
+        def deserializer(key, value):
+            call_record.append((key, value))
+            if key == 'z':
+                return state[key]  # scalar
+            value[...] = backend_config.device.send(state[key])
+            return value
+
+        l = chainer.Link()
+        with l.init_scope():
+            l.x = chainer.Parameter(shape=self.shape_x)
+            l.y = chainer.Parameter(shape=self.shape_y)
+        l.add_persistent('z', 1)
+        l.to_device(backend_config.device)
+
+        old_x_data = l.x.array
+        old_y_data = l.y.array
+        old_z = l.z
+
+        l.serialize(deserializer)
+
+        # Check link data
+        self.assertIs(l.x.array, old_x_data)
+        self.assertIs(l.y.array, old_y_data)
+        _assert_arrays_equal(l.x.array, state['x'])
+        _assert_arrays_equal(l.y.array, state['y'])
+        self.assertEqual(l.z, state['z'])
+
+        # Check inputs to the deserializer
+        self.assertEqual(len(call_record), 3)
+        call_record = sorted(call_record)
+        self.assertEqual(call_record[0][0], 'x')
+        self.assertIs(call_record[0][1], l.x.array)
+        self.assertEqual(call_record[1][0], 'y')
+        self.assertIs(call_record[1][1], l.y.array)
+        self.assertEqual(call_record[2][0], 'z')
+        self.assertEqual(call_record[2][1], old_z)
+
+    def test_deserialize_uninitialized1(self, backend_config):
+        # Deserializes uninitialized parameters into initialized ones.
+        # TODO(niboshi): Currently the existing initialized parameters are
+        # untouched, but maybe uninitialized state should be restored? (#7916)
+        call_record = []
+
+        def deserializer(key, value):
+            call_record.append((key, value))
+            return None  # to be uninitialized
+
+        l = chainer.Link()
+        with l.init_scope():
+            l.x = chainer.Parameter(shape=self.shape_x)  # initialized
+            l.y = chainer.Parameter(shape=self.shape_y)  # initialized
+        l.to_device(backend_config.device)
+
+        old_x_data = l.x.array
+        old_y_data = l.y.array
+
+        l.serialize(deserializer)
+
+        # Link is kept untouched
+        self.assertIs(l.x.array, old_x_data)
+        self.assertIs(l.y.array, old_y_data)
+
+        # Check inputs to the deserializer
+        self.assertEqual(len(call_record), 2)
+        call_record = sorted(call_record)
+        self.assertEqual(call_record[0][0], 'x')
+        self.assertIs(call_record[0][1], l.x.array)
+        self.assertEqual(call_record[1][0], 'y')
+
+    def test_deserialize_uninitialized2(self, backend_config):
+        # Deserializes initialized parameters into uninitialized ones.
+        call_record = []
+
+        state = {
+            'x': _shaped_random(self.shape_x, 'float32'),
+            'y': _shaped_random(self.shape_y, 'float32'),
+            'z': numpy.random.randn(),
+        }
+
+        def deserializer(key, value):
+            call_record.append((key, value))
+            # to be initialized
+            return backend_config.device.send(state[key])
+
+        l = chainer.Link()
+        with l.init_scope():
+            l.x = chainer.Parameter()  # uninitialized
+            l.y = chainer.Parameter()  # uninitialized
+        l.to_device(backend_config.device)
+
+        l.serialize(deserializer)
+
+        # Link is initialized
+        self.assertIsNotNone(l.x.array)
+        self.assertIsNotNone(l.y.array)
+        _assert_arrays_equal(l.x.array, state['x'])
+        _assert_arrays_equal(l.y.array, state['y'])
+
+        # Check inputs to the deserializer
+        self.assertEqual(len(call_record), 2)
+        call_record = sorted(call_record)
+        self.assertEqual(call_record[0][0], 'x')
+        self.assertIs(call_record[0][1], None)
+        self.assertEqual(call_record[1][0], 'y')
+        self.assertIs(call_record[1][1], None)
+
+
+@_inject_backend_tests
+class TestLinkSerializeDeserializedUninitializedParameter(unittest.TestCase):
+
+    def test_serialize(self, backend_config):
+        call_record = []
+
+        def serializer(key, value):
+            call_record.append((key, value))
+            return value
+
+        l = chainer.Link()
+        with l.init_scope():
+            l.x = chainer.Parameter()  # uninitialized
+        l.to_device(backend_config.device)
+
+        l.serialize(serializer)
+
+        # Link is kept uninitialized
+        self.assertIsNone(l.x.array)
+
+        # Check inputs to the serializer
+        self.assertEqual(len(call_record), 1)
+        self.assertEqual(call_record[0][0], 'x')
+        self.assertIs(call_record[0][1], None)
+
+    def test_deserialize(self, backend_config):
+        # Deserializes uninitialized parameters into uninitialied ones.
+        call_record = []
+
+        def serializer(key, value):
+            call_record.append((key, value))
+            return None  # to be uninitialized
+
+        l = chainer.Link()
+        with l.init_scope():
+            l.x = chainer.Parameter()  # uninitialized
+        l.to_device(backend_config.device)
+
+        l.serialize(serializer)
+
+        # Link is kept uninitialized
+        self.assertIsNone(l.x.array)
+
+        # Check inputs to the serializer
+        self.assertEqual(len(call_record), 1)
+        self.assertEqual(call_record[0][0], 'x')
+        self.assertIs(call_record[0][1], None)
+
+
+@_inject_backend_tests_no_intel64
 @attr.chainerx
 class TestLinkFromToChainerx(LinkTestBase, unittest.TestCase):
 
@@ -1267,19 +1462,7 @@ def test_count_params(self):
         assert not w
 
 
-@testing.backend.inject_backend_tests(
-    None,
-    [
-        # NumPy
-        {},
-        # CuPy
-        {'use_cuda': True, 'cuda_device': 0},
-        {'use_cuda': True, 'cuda_device': 1},
-        # ChainerX
-        {'use_chainerx': True, 'chainerx_device': 'native:0'},
-        {'use_chainerx': True, 'chainerx_device': 'cuda:0'},
-        {'use_chainerx': True, 'chainerx_device': 'cuda:1'},
-    ])
+@_inject_backend_tests_no_intel64
 @attr.chainerx
 class TestChainFromToChainerx(ChainTestBase, unittest.TestCase):
 
@@ -2481,19 +2664,7 @@ def test_to_intel64(self):
         assert l.child.to_method_called == 1
 
 
-@testing.backend.inject_backend_tests(
-    None,
-    [
-        # CPU
-        {},
-        # CUDA
-        {'use_cuda': True, 'cuda_device': 0},
-        {'use_cuda': True, 'cuda_device': 1},
-        # ChainerX
-        {'use_chainerx': True, 'chainerx_device': 'native:0'},
-        {'use_chainerx': True, 'chainerx_device': 'cuda:0'},
-        {'use_chainerx': True, 'chainerx_device': 'cuda:1'},
-    ])
+@_inject_backend_tests
 class TestSerialize(unittest.TestCase):
     def setUp(self):
         self.array = numpy.array([1, 2, 3], dtype=numpy.float32)