Repository: chainer/chainer

Issue:
Difference in the implementation of rmsprop  with Tensor Flow
There is a small difference between Chainer's implementation of rmsprop and the one in Tensor Flow which I thought worth pointing out. In [TF](https://github.com/tensorflow/tensorflow/blob/r1.8/tensorflow/python/training/rmsprop.py), the epsilon is inside the sqrt function while this is not the case for Chainer's [implementation](https://github.com/chainer/chainer/blob/v4.0.0/chainer/optimizers/rmsprop.py). This does not make much difference when you pick very small (compared to mean grad**2) values for epsilon but it does when epsilon is large (i.e eps=1.0) as it is used in training Inception [V3](https://arxiv.org/pdf/1512.00567.pdf) and [V4](https://arxiv.org/pdf/1602.07261v1.pdf). 


Discussion:
Thank you for pointing this out.

* Caffe2 (eps inside sqrt): https://github.com/pytorch/pytorch/blob/v0.4.0/caffe2/sgd/rmsprop_op_gpu.cu#L24
* PyTorch (eps outside sqrt): https://github.com/pytorch/pytorch/blob/v0.4.0/torch/optim/rmsprop.py#L93
After the discussion, we will provide `eps_inside_sqrt=False` option, which can be used to switch this behavior. The default value is `False` to keep backward compatibility, but users who expects the computation equivalent to TensorFlow or Caffe2 can change the option to `True`.

Resolving patch:
diff --git a/chainer/optimizers/rmsprop.py b/chainer/optimizers/rmsprop.py
--- a/chainer/optimizers/rmsprop.py
+++ b/chainer/optimizers/rmsprop.py
@@ -8,6 +8,7 @@
 _default_hyperparam.lr = 0.01
 _default_hyperparam.alpha = 0.99
 _default_hyperparam.eps = 1e-8
+_default_hyperparam.eps_inside_sqrt = False
 
 
 class RMSpropRule(optimizer.UpdateRule):
@@ -23,11 +24,19 @@ class RMSpropRule(optimizer.UpdateRule):
         lr (float): Learning rate.
         alpha (float): Exponential decay rate of the second order moment.
         eps (float): Small value for the numerical stability.
+        eps_inside_sqrt (bool): When ``True``, gradient will be divided by
+            :math:`\\sqrt{ms + eps}` where ``ms`` is the mean square. When
+            ``False`` (default), gradient will be divided by
+            :math:`\\sqrt{ms} + eps` instead.
+            This option may be convenient for users porting code from other
+            frameworks;
+            see `#4754 <https://github.com/chainer/chainer/issues/4754>`__ for
+            details.
 
     """
-    _kernel = None
 
-    def __init__(self, parent_hyperparam=None, lr=None, alpha=None, eps=None):
+    def __init__(self, parent_hyperparam=None, lr=None, alpha=None, eps=None,
+                 eps_inside_sqrt=None):
         super(RMSpropRule, self).__init__(
             parent_hyperparam or _default_hyperparam)
         if lr is not None:
@@ -36,6 +45,8 @@ def __init__(self, parent_hyperparam=None, lr=None, alpha=None, eps=None):
             self.hyperparam.alpha = alpha
         if eps is not None:
             self.hyperparam.eps = eps
+        if eps_inside_sqrt is not None:
+            self.hyperparam.eps_inside_sqrt = eps_inside_sqrt
 
     def init_state(self, param):
         xp = cuda.get_array_module(param.data)
@@ -56,7 +67,11 @@ def update_core_cpu(self, param):
 
         ms *= hp.alpha
         ms += (1 - hp.alpha) * grad * grad
-        param.data -= hp.lr * grad / (numpy.sqrt(ms) + eps)
+        if hp.eps_inside_sqrt:
+            denom = numpy.sqrt(ms + eps)
+        else:
+            denom = numpy.sqrt(ms) + eps
+        param.data -= hp.lr * grad / denom
 
     def update_core_gpu(self, param):
         grad = param.grad
@@ -68,15 +83,18 @@ def update_core_gpu(self, param):
             raise ValueError(
                 'eps of RMSprop optimizer is too small for {} ({})'.format(
                     grad.dtype.name, hp.eps))
-        if RMSpropRule._kernel is None:
-            RMSpropRule._kernel = cuda.elementwise(
-                'T grad, T lr, T alpha, T eps',
-                'T param, T ms',
-                '''ms = alpha * ms + (1 - alpha) * grad * grad;
-                   param -= lr * grad / (sqrt(ms) + eps);''',
-                'rmsprop')
-        RMSpropRule._kernel(grad, self.hyperparam.lr, self.hyperparam.alpha,
-                            eps, param.data, self.state['ms'])
+        if hp.eps_inside_sqrt:
+            denom = 'sqrt(ms + eps)'
+        else:
+            denom = 'sqrt(ms) + eps'
+        kernel = cuda.elementwise(
+            'T grad, T lr, T alpha, T eps',
+            'T param, T ms',
+            '''ms = alpha * ms + (1 - alpha) * grad * grad;
+               param -= lr * grad / ({});'''.format(denom),
+            'rmsprop')
+        kernel(grad, self.hyperparam.lr, self.hyperparam.alpha,
+               eps, param.data, self.state['ms'])
 
 
 class RMSprop(optimizer.GradientMethod):
@@ -90,19 +108,30 @@ class RMSprop(optimizer.GradientMethod):
         lr (float): Learning rate.
         alpha (float): Exponential decay rate of the second order moment.
         eps (float): Small value for the numerical stability.
+        eps_inside_sqrt (bool): When ``True``, gradient will be divided by
+            :math:`\\sqrt{ms + eps}` where ``ms`` is the mean square. When
+            ``False`` (default), gradient will be divided by
+            :math:`\\sqrt{ms} + eps` instead.
+            This option may be convenient for users porting code from other
+            frameworks;
+            see `#4754 <https://github.com/chainer/chainer/issues/4754>`__ for
+            details.
 
     """
 
     def __init__(self, lr=_default_hyperparam.lr,
-                 alpha=_default_hyperparam.alpha, eps=_default_hyperparam.eps):
+                 alpha=_default_hyperparam.alpha, eps=_default_hyperparam.eps,
+                 eps_inside_sqrt=_default_hyperparam.eps_inside_sqrt):
         super(RMSprop, self).__init__()
         self.hyperparam.lr = lr
         self.hyperparam.alpha = alpha
         self.hyperparam.eps = eps
+        self.hyperparam.eps_inside_sqrt = eps_inside_sqrt
 
     lr = optimizer.HyperparameterProxy('lr')
     alpha = optimizer.HyperparameterProxy('alpha')
     eps = optimizer.HyperparameterProxy('eps')
+    eps_inside_sqrt = optimizer.HyperparameterProxy('eps_inside_sqrt')
 
     def create_update_rule(self):
         return RMSpropRule(self.hyperparam)


Test patch:
diff --git a/tests/chainer_tests/optimizers_tests/test_optimizers_by_linear_model.py b/tests/chainer_tests/optimizers_tests/test_optimizers_by_linear_model.py
--- a/tests/chainer_tests/optimizers_tests/test_optimizers_by_linear_model.py
+++ b/tests/chainer_tests/optimizers_tests/test_optimizers_by_linear_model.py
@@ -237,14 +237,14 @@ def create(self):
 @testing.parameterize(*testing.product({
     'dtype': [numpy.float16, numpy.float32, numpy.float64],
     'use_placeholder': [False, True],
+    'eps_inside_sqrt': [False, True],
 }))
 class TestRMSprop(OptimizerTestBase, unittest.TestCase):
 
     def create(self):
+        kwargs = {'eps_inside_sqrt': self.eps_inside_sqrt}
         if self.dtype == numpy.float16:
-            kwargs = {'eps': 1e-6}
-        else:
-            kwargs = {}
+            kwargs['eps'] = 1e-6
         return optimizers.RMSprop(0.1, **kwargs)