Repository: chainer/chainer

Issue:
Optimizer with `use_fp32_update` does not work for uninitialized parameters
It was reported that it fails at the evaluation of `param.dtype`:
https://github.com/chainer/chainer/blob/v7.0.0a1/chainer/optimizer.py#L196


Discussion:

Resolving patch:
diff --git a/chainer/optimizer.py b/chainer/optimizer.py
--- a/chainer/optimizer.py
+++ b/chainer/optimizer.py
@@ -243,32 +243,67 @@ def update(self, param):
 
         self.t += 1
 
-        if self._use_fp32_update and param.dtype == numpy.float16:
+        try:
+            param_dtype = param.dtype
+        except RuntimeError:
+            param_dtype = None  # uninitialized and dtype is not determined
+
+        is_initialized = param.array is not None
+        loss_scale = param._loss_scale
+
+        # Apply use_fp32_update
+        if self._use_fp32_update and param_dtype == numpy.float16:
+            # Create fp32 parameter if not created yet.
             if self._fp32_param is None:
-                self._fp32_param = variable.Variable(
-                    param.array.astype(numpy.float32),
-                    name=param.name)
+                if is_initialized:
+                    self._fp32_param = variable.Parameter(
+                        param.array.astype(numpy.float32),
+                        name=param.name)
+                else:
+                    self._fp32_param = self._create_uninitialized_parameter(
+                        numpy.float32, name=param.name)
             fp32_param = self._fp32_param
-            fp32_param.grad = param.grad.astype(numpy.float32)
 
-            if fp32_param.data is not None:
-                self._prepare(fp32_param)
-            if param._loss_scale is not None:
-                fp32_param.grad /= param._loss_scale
-            self._hookable.call_hooks('pre', args=(self, fp32_param,))
-            self.update_core(fp32_param)
-            self._hookable.call_hooks('post', args=(self, fp32_param,))
+            # Convert the gradient
+            if is_initialized:
+                fp32_param.grad = param.grad.astype(numpy.float32)
 
-            param.data = fp32_param.data.astype(param.dtype)
-            fp32_param.grad = None
+            param_ = fp32_param
+            fp32_converted = True
         else:
-            if param.data is not None:
-                self._prepare(param)
-            if param._loss_scale is not None:
-                param.grad /= param._loss_scale
-            self._hookable.call_hooks('pre', args=(self, param,))
-            self.update_core(param)
-            self._hookable.call_hooks('post', args=(self, param,))
+            param_ = param
+            fp32_converted = False
+
+        if is_initialized:
+            # Init states
+            self._init_states(param_)
+
+            # Apply loss scaling
+            if loss_scale is not None:
+                param_.grad /= loss_scale
+
+        # Call update_core
+        self._hookable.call_hooks('pre', args=(self, param_,))
+        self.update_core(param_)
+        self._hookable.call_hooks('post', args=(self, param_,))
+
+        # Convert back to the original dtype
+        if fp32_converted:
+            if is_initialized:
+                param.array = fp32_param.array.astype(param.dtype)
+            fp32_param.grad = None
+
+    def _create_uninitialized_parameter(self, dtype, name):
+        # Creates an uninitialized parameter with given dtype.
+        # This is somewhat tricky but the parameter is created with a
+        # dummy initializer with the dtype.
+        def initializer(array):
+            assert False  # the parameter should never be initialized.
+        initializer.dtype = dtype
+        param = variable.Parameter(initializer, name=name)
+        assert param.dtype == dtype
+        assert param.array is None
+        return param
 
     def update_core(self, param):
         """Updates the parameter.
@@ -414,7 +449,7 @@ def serialize(self, serializer):
                         value = None
                     # leave the update rule state as `None` if the keys are not
                     # contained in the snapshot, so that these states can be
-                    # automatically initialized with the `_prepare` method
+                    # automatically initialized with the `_init_states` method
                     if value is None:
                         self._state = None
                         break
@@ -424,7 +459,7 @@ def serialize(self, serializer):
             for key in self._state:
                 self._state[key] = serializer(key, self._state[key])
 
-    def _prepare(self, param):
+    def _init_states(self, param):
         device = param.device
         with chainer.using_device(device):
             state = self.state
diff --git a/chainer/variable.py b/chainer/variable.py
--- a/chainer/variable.py
+++ b/chainer/variable.py
@@ -1700,6 +1700,19 @@ def __reduce__(self):
             self.initializer, self.update_rule, self.device)
         return _recover_parameter, args
 
+    @property
+    def dtype(self):
+        array = self.array
+        if array is not None:
+            return array.dtype
+        # uninitialized
+        initializer = self.initializer
+        if hasattr(initializer, 'dtype'):
+            return numpy.dtype(initializer.dtype)
+        raise RuntimeError(
+            'Dtype of the parameter is not determined yet because it\'s '
+            'uninitialized and dtype was not explicitly given.')
+
     def to_cpu(self):
         return self.to_device(backend.CpuDevice())
 


Test patch:
diff --git a/tests/chainer_tests/test_optimizer.py b/tests/chainer_tests/test_optimizer.py
--- a/tests/chainer_tests/test_optimizer.py
+++ b/tests/chainer_tests/test_optimizer.py
@@ -4,6 +4,7 @@
 
 import mock
 import numpy as np
+import pytest
 
 import chainer
 from chainer import backend
@@ -767,4 +768,67 @@ def test_weight_decay(self):
         self.assertIs(self.warnings[-1].category, DeprecationWarning)
 
 
+@testing.parameterize(*testing.product({
+    # None: dtype is not given by initializer.
+    # Otherwise: it's given by initializer.
+    'dtype': [None, np.float16, np.float32, np.float64]
+}))
+class TestUpdateRuleUseFp32Update(unittest.TestCase):
+
+    def test_uninitialized_parameter(self):
+        dtype = self.dtype
+
+        def initializer(array):
+            assert False  # never called
+
+        # Set initializer.dtype to specify the parameter's dtype
+        if dtype is not None:
+            initializer.dtype = dtype
+
+        # Create an uninitialized parameter
+        param = chainer.Parameter(initializer)
+        assert param.array is None
+        if dtype is not None:
+            assert param.dtype == dtype
+
+        # Create an update rule with custom update_core
+        record = []
+        update_rule = chainer.UpdateRule()
+
+        def update_core(param):
+            # param.dtype may not be retrieved because it can be uninitialized
+            # and dtype is not given (i.e. self.dtype is None)
+            try:
+                param_dtype = param.dtype
+            except RuntimeError:
+                param_dtype = None
+            record.append({
+                'param': param,
+                'dtype': param_dtype,
+            })
+
+        update_rule.update_core = update_core
+
+        # Enable fp32 update
+        update_rule.use_fp32_update()
+        # Call update_rule.update
+        update_rule.update(param)
+
+        if dtype == np.float16:
+            assert record[0]['param'] is not param
+            assert record[0]['dtype'] == np.float32
+        else:
+            assert record[0]['param'] is param
+            assert record[0]['dtype'] == dtype
+
+        # The original parameter is kept uninitialized and its dtype is
+        # unchanged.
+        assert param.array is None
+        if dtype is not None:
+            assert param.dtype == dtype
+        else:
+            with pytest.raises(RuntimeError):
+                param.dtype
+
+
 testing.run_module(__name__, __file__)
diff --git a/tests/chainer_tests/test_variable.py b/tests/chainer_tests/test_variable.py
--- a/tests/chainer_tests/test_variable.py
+++ b/tests/chainer_tests/test_variable.py
@@ -1989,6 +1989,25 @@ def test_addgrad_to_uninitialized_parameter_cpu_to_chx(self):
         # TODO(sonots): Support addgrad with ChainerX
         raise unittest.SkipTest('ChainerX does not support addgrad')
 
+    def test_dtype_given_by_initializer(self):
+        class MyInitializer(object):
+            dtype = 'float16'
+
+            def __call__(self, array):
+                assert False  # never called
+
+        param = chainer.Parameter(MyInitializer())
+        assert param.dtype == np.float16
+
+    def test_dtype_not_given(self):
+        class MyInitializer(object):
+            def __call__(self, array):
+                assert False  # never called
+
+        param = chainer.Parameter(MyInitializer())
+        with pytest.raises(RuntimeError):
+            param.dtype
+
 
 class TestDebugPrint(unittest.TestCase):