Repository: chainer/chainer

Issue:
Flaky test: `TestAdam`
https://jenkins.preferred.jp/job/chainer/job/chainer_pr/1490/TEST=chainer-py3,label=mn1-p100/

```
20:51:03 _ TestAdam_param_3_{adabound=False, amsgrad=False, dtype=float16, loss_scaling='static', use_placeholder=True}.test_linear_model_multi_gpu _
20:51:03 chainer/testing/parameterized.py:89: in new_method
20:51:03     utils._raise_from(e.__class__, s.getvalue(), e)
20:51:03 chainer/utils/__init__.py:106: in _raise_from
20:51:03     six.raise_from(new_exc.with_traceback(orig_exc.__traceback__), None)
20:51:03 chainer/testing/parameterized.py:78: in new_method
20:51:03     return base_method(self, *args, **kwargs)
20:51:03 chainer/testing/condition.py:80: in wrapper
20:51:03     fail()
20:51:03 chainer/testing/condition.py:51: in fail
20:51:03     instance.fail(msg)
20:51:03 E   AssertionError: Parameterized test failed.
20:51:03 E   
20:51:03 E   Base test method: TestAdam.test_linear_model_multi_gpu
20:51:03 E   Test parameters:
20:51:03 E     adabound: False
20:51:03 E     amsgrad: False
20:51:03 E     dtype: <class 'numpy.float16'>
20:51:03 E     loss_scaling: static
20:51:03 E     use_placeholder: True
20:51:03 E   
20:51:03 E   
20:51:03 E   (caused by)
20:51:03 E   AssertionError: 
20:51:03 E   Fail: 10, Success: 0
20:51:03 E   
20:51:03 E   The first error message:
20:51:03 E   Traceback (most recent call last):
20:51:03 E     File "/work/chainer/chainer/testing/condition.py", line 60, in <lambda>
20:51:03 E       lambda: f(ins, *args[1:], **kwargs),
20:51:03 E     File "/work/chainer/tests/chainer_tests/optimizers_tests/test_optimizers_by_linear_model.py", line 168, in test_linear_model_multi_gpu
20:51:03 E       self.assertGreater(cuda.to_cpu(accuracy.data), 0.9)
20:51:03 E   AssertionError: array(0.5, dtype=float16) not greater than 0.9
```


Discussion:
Seems that the error is related to `float16`, I changed the `@condition.retry(10)` to `@condition.retry(1). This allowed me to reproduce the failures in my environment and it only fails when float16 is used.
Maybe there is the need to use mixed-precission inside the classifier?
FYI, I observed this error yesterday twice in a row.
Before that I have never seen it. It could be related to some recent change.

Resolving patch:
diff --git a/chainer/optimizers/adam.py b/chainer/optimizers/adam.py
--- a/chainer/optimizers/adam.py
+++ b/chainer/optimizers/adam.py
@@ -240,12 +240,18 @@ def update_core_gpu(self, param):
                     'T eps, T eta, T weight_decay_rate, raw T dummy',
                     'P param, P m, P v, P vhat',
                     '''T grad_ = static_cast<T>(grad);
-                       m += one_minus_beta1 * (grad_ - m);
-                       v += one_minus_beta2 * (grad_ * grad - v);
-                       vhat = max(vhat, v);
+                       T m_ = static_cast<T>(m);
+                       T v_ = static_cast<T>(v);
+                       T vhat_ = static_cast<T>(vhat);
+                       m_ += one_minus_beta1 * (grad_ - m_);
+                       v_ += one_minus_beta2 * (grad_ * grad_ - v_);
+                       vhat_ = max(vhat_, v_);
+                       vhat = static_cast<T>(vhat_);
+                       m = static_cast<P>(m_);
+                       v = static_cast<P>(v_);
                        param -= eta *
-                           (max(min(alpha_t / (sqrt(vhat) + eps), upper),
-                                lower) * m + weight_decay_rate * param);''',
+                           (max(min(alpha_t / (sqrt(vhat_) + eps), upper),
+                                lower) * m_ + weight_decay_rate * param);''',
                     'amsbound')
             AdamRule._amsbound_kernel(
                 grad, self.alpha_t, 1 - hp.beta1,
@@ -261,11 +267,15 @@ def update_core_gpu(self, param):
                     'T eps, T eta, T weight_decay_rate, raw T dummy',
                     'P param, P m, P v',
                     '''T grad_ = static_cast<T>(grad);
-                       m += one_minus_beta1 * (grad_ - m);
-                       v += one_minus_beta2 * (grad_ * grad_ - v);
+                       T m_ = static_cast<T>(m);
+                       T v_ = static_cast<T>(v);
+                       m_ += one_minus_beta1 * (grad_ - m_);
+                       v_ += one_minus_beta2 * (grad_ * grad_ - v_);
+                       m = static_cast<P>(m_);
+                       v = static_cast<P>(v_);
                        param -= eta *
-                           (max(min(alpha_t / (sqrt(v) + eps), upper),
-                                lower) * m + weight_decay_rate * param);''',
+                           (max(min(alpha_t / (sqrt(v_) + eps), upper),
+                                lower) * m_ + weight_decay_rate * param);''',
                     'adabound')
             AdamRule._adabound_kernel(
                 grad, self.alpha_t, 1 - hp.beta1,
@@ -279,10 +289,16 @@ def update_core_gpu(self, param):
                     'T eps, T eta, T weight_decay_rate, raw T dummy',
                     'P param, P m, P v, P vhat',
                     '''T grad_ = static_cast<T>(grad);
-                       m += one_minus_beta1 * (grad_ - m);
-                       v += one_minus_beta2 * (grad_ * grad_ - v);
-                       vhat = max(vhat, v);
-                       param -= eta * (alpha_t * m / (sqrt(vhat) + eps) +
+                       T m_ = static_cast<T>(m);
+                       T v_ = static_cast<T>(v);
+                       T vhat_ = static_cast<T>(vhat);
+                       m_ += one_minus_beta1 * (grad_ - m_);
+                       v_ += one_minus_beta2 * (grad_ * grad_ - v_);
+                       vhat_ = max(vhat_, v_);
+                       vhat = static_cast<T>(vhat_);
+                       m = static_cast<P>(m_);
+                       v = static_cast<P>(v_);
+                       param -= eta * (alpha_t * m_ / (sqrt(vhat_) + eps) +
                                        weight_decay_rate * param);''',
                     'adam')
             AdamRule._amsgrad_kernel(
@@ -298,9 +314,13 @@ def update_core_gpu(self, param):
                     'T eps, T eta, T weight_decay_rate, raw T dummy',
                     'P param, P m, P v',
                     '''T grad_ = static_cast<T>(grad);
-                       m += one_minus_beta1 * (grad_ - m);
-                       v += one_minus_beta2 * (grad_ * grad_ - v);
-                       param -= eta * (alpha_t * m / (sqrt(v) + eps) +
+                       T m_ = static_cast<T>(m);
+                       T v_ = static_cast<T>(v);
+                       m_ += one_minus_beta1 * (grad_ - m_);
+                       v_ += one_minus_beta2 * (grad_ * grad_ - v_);
+                       m = static_cast<P>(m_);
+                       v = static_cast<P>(v_);
+                       param -= eta * (alpha_t * m_ / (sqrt(v_) + eps) +
                                        weight_decay_rate * param);''',
                     'adam')
             AdamRule._kernel(


Test patch: