Repository: chainer/chainer

Issue:
`eps` is doubly added to variance in inference of `L.BatchRenormalization`
- `runninng_var` learns variances with `eps`
- `train=False` mode uses `running_var + eps`

### Conditions
I tested with Chainer versions: `3.0.0`, `4.5.0`, `5.4.0`, `7.0.0a1`

### Code to reproduce
```python
import chainer
import numpy as np
np.random.seed(0)

brn = chainer.links.BatchRenormalization(3, eps=1.)
for _ in range(1000):
    x = np.random.randn(1000, 3).astype('f')
    brn(x)

x = np.random.randn(1000, 3).astype('f')

y = brn(x)
print(y.array.var(axis=0))

with chainer.using_config('train', False):
    y = brn(x)
print(y.array.var(axis=0))
```

### Error messages, stack traces, or logs
```
[0.51281106 0.49953052 0.48790243]
[0.3506052  0.33283928 0.31892547]
```

Here, the "normalized" variance is around 1/(1+eps) if train, whereas it's around 1/(1+2 eps) otherwise.


Discussion:

Resolving patch:
diff --git a/chainer/functions/normalization/batch_renormalization.py b/chainer/functions/normalization/batch_renormalization.py
--- a/chainer/functions/normalization/batch_renormalization.py
+++ b/chainer/functions/normalization/batch_renormalization.py
@@ -76,8 +76,8 @@ def forward(self, inputs):
         # batch renormalization
         axis = (0,) + tuple(range(head_ndim, x.ndim))
         mean = x.mean(axis=axis, dtype=gamma.dtype)
-        var = x.var(axis=axis, dtype=gamma.dtype) + self.eps
-        self.std = xp.sqrt(var, dtype=var.dtype)
+        var = x.var(axis=axis, dtype=gamma.dtype)
+        self.std = xp.sqrt(var + self.eps, dtype=var.dtype)
 
         running_sigma = xp.sqrt(self._running_var + self.eps,
                                 dtype=self._running_mean.dtype)


Test patch:
diff --git a/tests/chainer_tests/functions_tests/normalization_tests/test_batch_renormalization.py b/tests/chainer_tests/functions_tests/normalization_tests/test_batch_renormalization.py
--- a/tests/chainer_tests/functions_tests/normalization_tests/test_batch_renormalization.py
+++ b/tests/chainer_tests/functions_tests/normalization_tests/test_batch_renormalization.py
@@ -45,7 +45,7 @@ def _naive_batch_renormalization(
         m = x.size // gamma.size
         adjust = m / max(m - 1., 1.)
         running_var *= decay
-        running_var += (var.array + dt(eps)) * dt((1. - decay) * adjust)
+        running_var += var.array * dt((1. - decay) * adjust)
 
     return y
 
diff --git a/tests/chainer_tests/links_tests/normalization_tests/test_batch_renormalization.py b/tests/chainer_tests/links_tests/normalization_tests/test_batch_renormalization.py
--- a/tests/chainer_tests/links_tests/normalization_tests/test_batch_renormalization.py
+++ b/tests/chainer_tests/links_tests/normalization_tests/test_batch_renormalization.py
@@ -117,7 +117,8 @@ def test_forward_multi_gpu(self):
 
 
 @testing.parameterize(
-    {'nx': 10, 'ny': 10},
+    {'nx': 10, 'ny': 10, 'eps': 2e-5},
+    {'nx': 10, 'ny': 10, 'eps': 1e-1},
     # TODO(Kenta Oono)
     # Pass the case below (this test does not pass when nx != ny).
     # {'nx': 10, 'ny': 15}
@@ -127,7 +128,8 @@ class TestPopulationStatistics(unittest.TestCase):
     def setUp(self):
         self.decay = 0.9
         self.size = 3
-        self.link = links.BatchRenormalization(self.size, self.decay)
+        self.link = links.BatchRenormalization(
+            self.size, decay=self.decay, eps=self.eps)
         self.x = numpy.random.uniform(
             -1, 1, (self.nx, self.size)).astype(numpy.float32)
         self.y = numpy.random.uniform(
@@ -141,18 +143,16 @@ def check_statistics(self, x, y):
         unbiased_var = self.x.var(axis=0) * self.nx / (self.nx - 1)
         testing.assert_allclose(unbiased_var, self.link.avg_var)
 
+        y = chainer.Variable(y)
         with chainer.using_config('train', False):
-            y = chainer.Variable(y)
             self.link(y, finetune=True)
-            testing.assert_allclose(mean, self.link.avg_mean)
-            testing.assert_allclose(unbiased_var, self.link.avg_var)
+        testing.assert_allclose(mean, self.link.avg_mean)
+        testing.assert_allclose(unbiased_var, self.link.avg_var)
 
-    @condition.retry(3)
     def test_statistics_cpu(self):
         self.check_statistics(self.x, self.y)
 
     @attr.gpu
-    @condition.retry(3)
     def test_statistics_gpu(self):
         self.link.to_gpu()
         self.check_statistics(cuda.to_gpu(self.x), cuda.to_gpu(self.y))
@@ -176,12 +176,10 @@ def check_statistics2(self, x, y):
         testing.assert_allclose(mean, self.link.avg_mean)
         testing.assert_allclose(unbiased_var, self.link.avg_var)
 
-    @condition.retry(3)
     def test_statistics2_cpu(self):
         self.check_statistics2(self.x, self.y)
 
     @attr.gpu
-    @condition.retry(3)
     def test_statistics2_gpu(self):
         self.link.to_gpu()
         self.check_statistics2(