Repository: chainer/chainer

Issue:
BatchNormalization's avg_var initialized with zeros can cause division by zero
Currently, `BatchNormalization`'s `avg_var` is initialized with zeros. This will cause division by zero when called it with test=True before test=False, so it might be better we use another values, say, ones, to initialize `avg_var`.

Calling `BatchNormalization` with test=True before test=False can seem unnatural, but it can happen in the reinforcement learning context: an agent selects an action by evaluating its value function or policy represented by a NN, observes consequences, and then (if it has enough experiences) updates its value function or policy.



Discussion:
This issue has been automatically marked as stale because it has not had recent activity. It will be closed after 30 days if no further activity occurs. Thank you for your contributions.

This issue is closed as announced. Feel free to re-open it if needed.

@muupan Do you think we should reopen this issue?
I think it is good to fix this issue by initializing var with 1s. PyTorch seems doing so.
https://github.com/pytorch/pytorch/blob/master/torch/nn/modules/batchnorm.py#L24

Are there any opinions against doing so?
One concern is that it changes the models learned with the same script. I think the effect is negligible for most use cases (the effect is made exponentially small against the number of training iterations), but it would be better to leave a way to reproduce the current behavior.
One simple solution would be to add an option such as `initialize_population_variance_with_zero` to `__init__`. Or more generally, we can give an initializer for initializing `avg_var` (and possibly `avg_mean`), although initializers are currently used for not persistent values but parameters.

How do you think of them?

+1 for using initializers for `avg_var` (and `avg_mean` for consistency).
@muupan How about you?
👍 
I'd rather not add any options.  Just fix it, please.  Even if someone exploited the fact that `avg_var` is initialized with zeros, he or she can fix code easily by
```
bn = chainer.links.BatchNormalization(_)
bn.avg_var[...] = 0
```
(and all Chainer has to do is to write it in the version-up guide).  To anyone else, including beginners, `avg_var` should not be 0.
I will work on it.

Resolving patch:
diff --git a/chainer/initializers/__init__.py b/chainer/initializers/__init__.py
--- a/chainer/initializers/__init__.py
+++ b/chainer/initializers/__init__.py
@@ -19,7 +19,7 @@
 from chainer.initializers.uniform import Uniform  # NOQA
 
 
-def generate_array(initializer, shape, xp):
+def generate_array(initializer, shape, xp, dtype=None):
     """Return initialized array.
 
     The algorithms used to make the new values depend on the
@@ -32,12 +32,20 @@ def generate_array(initializer, shape, xp):
              or :class:`cupy.ndarray` and edits its value.
         shape (tuple): Shape of a return array.
         xp (module): :mod:`cupy` or :mod:`numpy`.
+        dtype: Dtype specifier. If omitted, ``initializer.dtype`` is used.
 
     Returns:
         numpy.ndarray or cupy.ndarray: An initialized array.
 
     """
-    dtype = chainer.get_dtype(getattr(initializer, 'dtype', None))
+    dtype_attr = getattr(initializer, 'dtype', None)
+    if dtype is not None and dtype_attr is not None \
+            and numpy.dtype(dtype) != numpy.dtype(dtype_attr):
+        raise ValueError(
+            'dtype mismatch: {} != {}'.format(dtype, dtype_attr))
+    if dtype is None:
+        dtype = dtype_attr
+    dtype = chainer.get_dtype(dtype)
     array = xp.empty(shape, dtype=dtype)
     initializer(array)
     return array
diff --git a/chainer/links/normalization/batch_normalization.py b/chainer/links/normalization/batch_normalization.py
--- a/chainer/links/normalization/batch_normalization.py
+++ b/chainer/links/normalization/batch_normalization.py
@@ -53,6 +53,22 @@ class BatchNormalization(link.Link):
             option, numbers in the tuple must be being sorted in ascending
             order. For example, (0, 2) is OK, but (2, 0) is not.
 
+        initial_gamma: Initializer of the scaling parameter. The default value
+            is ``1``.
+        initial_beta: Initializer of the shifting parameter. The default value
+            is ``0``.
+        initial_avg_mean: Initializer of the moving average of population mean.
+            The default value is ``0``.
+        initial_avg_var: Initializer of the moving average of population
+            variance. The default value is ``1``.
+
+    .. note::
+
+        From v5.0.0, the initial value of the population variance is changed to
+        1. It does not change the behavior of training, but the resulting model
+        may have a slightly different behavior on inference. To emulate the
+        old behavior, pass ``initial_avg_var=0`` for training.
+
     See: `Batch Normalization: Accelerating Deep Network Training by Reducing\
           Internal Covariate Shift <https://arxiv.org/abs/1502.03167>`_
 
@@ -171,11 +187,14 @@ class BatchNormalization(link.Link):
 
     def __init__(self, size=None, decay=0.9, eps=2e-5, dtype=None,
                  use_gamma=True, use_beta=True,
-                 initial_gamma=None, initial_beta=None, axis=None):
+                 initial_gamma=None, initial_beta=None, axis=None,
+                 initial_avg_mean=None, initial_avg_var=None):
         super(BatchNormalization, self).__init__()
 
         if size is None and axis is None:
             raise RuntimeError('size or axis is required')
+        self._initial_avg_mean = initial_avg_mean
+        self._initial_avg_var = initial_avg_var
         self.N = 0
         self.register_persistent('N')
         self.decay = decay
@@ -204,9 +223,12 @@ def __init__(self, size=None, decay=0.9, eps=2e-5, dtype=None,
             self._initialize_params(size)
 
     def _initialize_params(self, shape):
-        self.avg_mean = numpy.zeros(shape, dtype=self._dtype)
+        dtype = self._dtype
+        self.avg_mean = _init_array(self._initial_avg_mean, 0, shape, dtype)
+        self._initial_avg_mean = None
         self.register_persistent('avg_mean')
-        self.avg_var = numpy.zeros(shape, dtype=self._dtype)
+        self.avg_var = _init_array(self._initial_avg_var, 1, shape, dtype)
+        self._initial_avg_var = None
         self.register_persistent('avg_var')
         if self.gamma is not None:
             self.gamma.initialize(shape)
@@ -288,3 +310,10 @@ def start_finetuning(self):
 
         """
         self.N = 0
+
+
+def _init_array(initializer, default_value, size, dtype):
+    if initializer is None:
+        initializer = default_value
+    initializer = initializers._get_initializer(initializer)
+    return initializers.generate_array(initializer, size, numpy, dtype=dtype)
diff --git a/chainer/links/normalization/batch_renormalization.py b/chainer/links/normalization/batch_renormalization.py
--- a/chainer/links/normalization/batch_renormalization.py
+++ b/chainer/links/normalization/batch_renormalization.py
@@ -28,10 +28,11 @@ class BatchRenormalization(BatchNormalization):
 
     def __init__(self, size, rmax=1, dmax=0, decay=0.9, eps=2e-5,
                  dtype=None, use_gamma=True, use_beta=True,
-                 initial_gamma=None, initial_beta=None):
-        super(BatchRenormalization, self).__init__(size, decay, eps, dtype,
-                                                   use_gamma, use_beta,
-                                                   initial_gamma, initial_beta)
+                 initial_gamma=None, initial_beta=None,
+                 initial_avg_mean=None, initial_avg_var=None):
+        super(BatchRenormalization, self).__init__(
+            size, decay, eps, dtype, use_gamma, use_beta,
+            initial_gamma, initial_beta, initial_avg_mean, initial_avg_var)
         self.rmax = rmax  # maximum allowed correction of variance
         self.dmax = dmax  # maximum allowed correction of mean
         self.r = None


Test patch:
diff --git a/tests/chainer_tests/links_tests/normalization_tests/test_batch_normalization.py b/tests/chainer_tests/links_tests/normalization_tests/test_batch_normalization.py
--- a/tests/chainer_tests/links_tests/normalization_tests/test_batch_normalization.py
+++ b/tests/chainer_tests/links_tests/normalization_tests/test_batch_normalization.py
@@ -317,6 +317,10 @@ def test_backward_gpu_without_cudnn(self):
             self.test_backward_gpu()
 
 
+def _generate_uniform(low, high, shape, dtype=numpy.float32):
+    return numpy.random.uniform(low, high, shape).astype(dtype)
+
+
 @testing.parameterize(*testing.product({
     'size': [3, (2, 3)],
 }))
@@ -324,18 +328,24 @@ class TestInitialize(unittest.TestCase):
 
     def setUp(self):
         self.decay = 0.9
-        self.initial_gamma = numpy.random.uniform(-1, 1, self.size)
-        self.initial_gamma = self.initial_gamma.astype(numpy.float32)
-        self.initial_beta = numpy.random.uniform(-1, 1, self.size)
-        self.initial_beta = self.initial_beta.astype(numpy.float32)
-        self.link = links.BatchNormalization(self.size, self.decay,
-                                             initial_gamma=self.initial_gamma,
-                                             initial_beta=self.initial_beta)
+        self.initial_gamma = _generate_uniform(-1, 1, self.size)
+        self.initial_beta = _generate_uniform(-1, 1, self.size)
+        self.initial_avg_mean = _generate_uniform(-1, 1, self.size)
+        self.initial_avg_var = _generate_uniform(-1, 1, self.size)
+        self.link = links.BatchNormalization(
+            self.size, self.decay,
+            initial_gamma=self.initial_gamma,
+            initial_beta=self.initial_beta,
+            initial_avg_mean=self.initial_avg_mean,
+            initial_avg_var=self.initial_avg_var,
+        )
 
     @condition.retry(3)
     def test_initialize_cpu(self):
         testing.assert_allclose(self.initial_gamma, self.link.gamma.data)
         testing.assert_allclose(self.initial_beta, self.link.beta.data)
+        testing.assert_allclose(self.initial_avg_mean, self.link.avg_mean)
+        testing.assert_allclose(self.initial_avg_var, self.link.avg_var)
 
     @attr.gpu
     @condition.retry(3)
@@ -343,6 +353,8 @@ def test_initialize_gpu(self):
         self.link.to_gpu()
         testing.assert_allclose(self.initial_gamma, self.link.gamma.data)
         testing.assert_allclose(self.initial_beta, self.link.beta.data)
+        testing.assert_allclose(self.initial_avg_mean, self.link.avg_mean)
+        testing.assert_allclose(self.initial_avg_var, self.link.avg_var)
 
 
 @testing.parameterize(*testing.product({
@@ -365,6 +377,8 @@ def setUp(self):
     def check_initialize(self):
         testing.assert_allclose(numpy.ones(self.size), self.link.gamma.array)
         testing.assert_allclose(numpy.zeros(self.size), self.link.beta.array)
+        testing.assert_allclose(0, self.link.avg_mean)
+        testing.assert_allclose(1, self.link.avg_var)
         y = self.link(self.x)
         assert y.dtype == self.dtype