Repository: chainer/chainer

Issue:
BatchNormalization with lazy initialization fail on GPU
# Problem description

```python
import chainer
import numpy as np

x = np.arange(12).reshape(4, 3).astype(np.float32) ** 2
x = chainer.cuda.to_gpu(x)
x = chainer.Variable(x)

# lazy initialization by not providing size
bn = chainer.links.BatchNormalization(axis=0)
bn.to_gpu()

bn(x)
```

This fails with following error:

```
---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-16-3375da8069af> in <module>()
----> 1 bn(x)

/usr/local/lib/python3.6/dist-packages/chainer/link.py in __call__(self, *args, **kwargs)
    240         if forward is None:
    241             forward = self.forward
--> 242         out = forward(*args, **kwargs)
    243 
    244         # Call forward_postprocess hook

/usr/local/lib/python3.6/dist-packages/chainer/links/normalization/batch_normalization.py in forward(self, x, **kwargs)
    297             ret = functions.batch_normalization(
    298                 x, gamma, beta, eps=self.eps, running_mean=self.avg_mean,
--> 299                 running_var=self.avg_var, decay=decay, axis=self.axis)
    300         else:
    301             # Use running average statistics or fine-tuned statistics.

/usr/local/lib/python3.6/dist-packages/chainer/functions/normalization/batch_normalization.py in batch_normalization(x, gamma, beta, **kwargs)
    830 
    831     return BatchNormalization(eps, running_mean, running_var, decay,
--> 832                               axis).apply((x, gamma, beta))[0]
    833 
    834 

/usr/local/lib/python3.6/dist-packages/chainer/function_node.py in apply(self, inputs)
    261                 outputs = static_forward_optimizations(self, in_data)
    262             else:
--> 263                 outputs = self.forward(in_data)
    264 
    265         # Check for output array types

/usr/local/lib/python3.6/dist-packages/chainer/functions/normalization/batch_normalization.py in forward(self, inputs)
    222                 x_desc.value, x.data.ptr, x_desc.value,
    223                 y.data.ptr, derivedBnDesc.value, gamma.data.ptr,
--> 224                 beta.data.ptr, factor, running_mean.data.ptr,
    225                 running_var.data.ptr, self.eps,
    226                 self.mean.data.ptr, self.inv_std.data.ptr)

AttributeError: 'memoryview' object has no attribute 'ptr'
```

Environment

```
Platform: Linux-4.14.65+-x86_64-with-Ubuntu-18.04-bionic
Chainer: 5.0.0
NumPy: 1.14.6
CuPy:
  CuPy Version          : 5.0.0
  CUDA Root             : /usr/local/cuda
  CUDA Build Version    : 9020
  CUDA Driver Version   : 9020
  CUDA Runtime Version  : 9020
  cuDNN Build Version   : 7201
  cuDNN Version         : 7201
  NCCL Build Version    : 2213
iDeep: Not Available
```

# Reason for failure

The problem was that `running_mean` was numpy array where it should've been a cupy array.

I figured out the problem lies in BatchNormalization's initializer. https://github.com/chainer/chainer/blob/v5.0.0/chainer/links/normalization/batch_normalization.py#L323

```python
def _init_array(initializer, default_value, size, dtype):
    if initializer is None:
        initializer = default_value
    initializer = initializers._get_initializer(initializer)
    return initializers.generate_array(initializer, size, numpy, dtype=dtype)
```

As you can see, `numpy` is hard coded in initializer.

I will send a pull request shortly.


Discussion:

Resolving patch:
diff --git a/chainer/links/normalization/batch_normalization.py b/chainer/links/normalization/batch_normalization.py
--- a/chainer/links/normalization/batch_normalization.py
+++ b/chainer/links/normalization/batch_normalization.py
@@ -1,5 +1,3 @@
-import numpy
-
 import chainer
 from chainer.backends import cuda
 from chainer import configuration
@@ -227,11 +225,10 @@ def __init__(self, size=None, decay=0.9, eps=2e-5, dtype=None,
             self._initialize_params(size)
 
     def _initialize_params(self, shape):
-        dtype = self._dtype
-        self.avg_mean = _init_array(self._initial_avg_mean, 0, shape, dtype)
+        self.avg_mean = self._init_array(self._initial_avg_mean, 0, shape)
         self._initial_avg_mean = None
         self.register_persistent('avg_mean')
-        self.avg_var = _init_array(self._initial_avg_var, 1, shape, dtype)
+        self.avg_var = self._init_array(self._initial_avg_var, 1, shape)
         self._initial_avg_var = None
         self.register_persistent('avg_var')
         if self.gamma is not None:
@@ -239,6 +236,13 @@ def _initialize_params(self, shape):
         if self.beta is not None:
             self.beta.initialize(shape)
 
+    def _init_array(self, initializer, default_value, size):
+        if initializer is None:
+            initializer = default_value
+        initializer = initializers._get_initializer(initializer)
+        return initializers.generate_array(
+            initializer, size, self.xp, dtype=self._dtype)
+
     def forward(self, x, **kwargs):
         """forward(self, x, finetune=False)
 
@@ -314,10 +318,3 @@ def start_finetuning(self):
 
         """
         self.N = 0
-
-
-def _init_array(initializer, default_value, size, dtype):
-    if initializer is None:
-        initializer = default_value
-    initializer = initializers._get_initializer(initializer)
-    return initializers.generate_array(initializer, size, numpy, dtype=dtype)


Test patch:
diff --git a/tests/chainer_tests/links_tests/normalization_tests/test_batch_normalization.py b/tests/chainer_tests/links_tests/normalization_tests/test_batch_normalization.py
--- a/tests/chainer_tests/links_tests/normalization_tests/test_batch_normalization.py
+++ b/tests/chainer_tests/links_tests/normalization_tests/test_batch_normalization.py
@@ -462,7 +462,7 @@ def test_no_inference(self):
         assert bn.avg_mean is not None
         assert bn.avg_var is not None
 
-    def test_inference(self):
+    def test_inference_cpu(self):
         bn = links.BatchNormalization(axis=self.axis)
         bn(self.x)
         assert bn.beta.shape == self.expected_size
@@ -470,6 +470,20 @@ def test_inference(self):
         assert bn.avg_mean.shape == self.expected_size
         assert bn.avg_var.shape == self.expected_size
 
+    @attr.gpu
+    def test_inference_gpu(self):
+        bn = links.BatchNormalization(axis=self.axis)
+        bn.to_gpu()
+        bn(cuda.to_gpu(self.x))
+        assert isinstance(bn.beta.data, cuda.cupy.ndarray)
+        assert isinstance(bn.gamma.data, cuda.cupy.ndarray)
+        assert isinstance(bn.avg_mean, cuda.cupy.ndarray)
+        assert isinstance(bn.avg_var, cuda.cupy.ndarray)
+        assert bn.beta.shape == self.expected_size
+        assert bn.gamma.shape == self.expected_size
+        assert bn.avg_mean.shape == self.expected_size
+        assert bn.avg_var.shape == self.expected_size
+
     def test_no_gamma(self):
         bn = links.BatchNormalization(axis=self.axis, use_gamma=False)
         assert bn.gamma is None