Repository: chainer/chainer

Issue:
`AveragePoolingNDGrad._get_pooling_width` is not implemented
In `AveragePoolingNDGrad`, `self._get_pooling_width` is used (I guess it should be `self.apoolnd._get_pooling_width` instead) when `self.pad_value` is None while
the class does not define the method and the tests of `F.average_pooling_nd` succeed.

Current `TestAveragePoolingND` does parameterize `pad_value` but does not use it in forward computation for backward tests. The default value of pad_value in F.average_pooling_nd is 0.

https://github.com/chainer/chainer/blob/d7b9bfd0e77fd79d58c4ae04cc6a1ac37ce66e8a/chainer/functions/pooling/average_pooling_nd.py#L184-L185
https://github.com/chainer/chainer/blob/d7b9bfd0e77fd79d58c4ae04cc6a1ac37ce66e8a/chainer/functions/pooling/average_pooling_nd.py#L201-L202

https://github.com/chainer/chainer/blob/d7b9bfd0e77fd79d58c4ae04cc6a1ac37ce66e8a/tests/chainer_tests/functions_tests/pooling_tests/test_average_pooling_nd.py#L20-L24
https://github.com/chainer/chainer/blob/d7b9bfd0e77fd79d58c4ae04cc6a1ac37ce66e8a/tests/chainer_tests/functions_tests/pooling_tests/test_average_pooling_nd.py#L140-L148

---

(2019/02/13): Backward tests fail if I just copy `_get_pooling_width` from `AveragePoolingND`.


Discussion:

Resolving patch:
diff --git a/chainer/functions/pooling/average_pooling_nd.py b/chainer/functions/pooling/average_pooling_nd.py
--- a/chainer/functions/pooling/average_pooling_nd.py
+++ b/chainer/functions/pooling/average_pooling_nd.py
@@ -5,6 +5,7 @@
 import six
 
 import chainer
+from chainer import backend
 from chainer.backends import cuda
 from chainer import function_node
 from chainer.functions.pooling import average_pooling_nd_kernel
@@ -103,6 +104,7 @@ def forward_cpu(self, inputs):
             dims = x.shape[2:]
             width = self._get_pooling_width(numpy, dims, x.dtype)
             y = col.sum(axis=y_axis) / width
+            self.width = width
         else:
             assert self.pad_value == 0
             y = col.mean(axis=y_axis)
@@ -132,6 +134,7 @@ def forward_gpu(self, inputs):
         if self.pad_value is None:
             coeff = self._get_pooling_width(cuda.cupy, idims, x.dtype)
             coeff = cuda.cupy.reciprocal(coeff, out=coeff)
+            self.coeff = coeff
         else:
             assert self.pad_value == 0
             coeff = 1. / functools.reduce(operator.mul, self.ksize)
@@ -177,14 +180,15 @@ def forward_cpu(self, gys):
         idims = self._in_shape[2:]
         odims = gy.shape[2:]
         colon = slice(None, None, None)
+        is_pad_value_none = self.pad_value is None
+        if is_pad_value_none:
+            width = self.apoolnd.width
+            numpy.divide(gy, width, out=gy)
         gy_index = (colon, colon) + (None,) * len(idims)
         gcol_reps = (1, 1) + self.ksize + (1,) * len(odims)
         gcol = numpy.tile(gy[gy_index], gcol_reps)
         gx = conv_nd.col2im_nd_cpu(gcol, self.stride, self.pad, idims)
-        if self.pad_value is None:
-            width = self._get_pooling_width(numpy, odims, gx.dtype)
-            numpy.divide(gx, width, out=gx)
-        else:
+        if not is_pad_value_none:
             gx /= functools.reduce(operator.mul, self.ksize)
         return gx,
 
@@ -193,16 +197,19 @@ def forward_gpu(self, gys):
             x, = self.apoolnd.get_retained_inputs()
             return self.apoolnd.backward_gpu((x.data,), gys)
 
+        is_pad_value_none = self.pad_value is None
+
         gy, = gys
         n, c = self._in_shape[:2]
         idims = self._in_shape[2:]
         odims = gy.shape[2:]
+        if is_pad_value_none:
+            coeff = self.apoolnd.coeff
+            # This conversion from chainerx to cupy exists here for
+            # double backward of chainerx on cuda.
+            coeff = backend.from_chx(coeff)
+            gy *= coeff
         gx = cuda.cupy.empty(self._in_shape, self._in_dtype)
-        if self.pad_value is None:
-            coeff = self._get_pooling_width(cuda.cupy, odims, gy.dtype)
-            coeff = cuda.cupy.reciprocal(coeff, out=coeff)
-        else:
-            coeff = 1. / functools.reduce(operator.mul, self.ksize)
 
         in_params, out_params, operation, name = \
             average_pooling_nd_kernel.AveragePoolingNDKernelBackward.generate(
@@ -210,14 +217,16 @@ def forward_gpu(self, gys):
         cuda.elementwise(in_params, out_params, operation, name)(
             gy.reduced_view(),
             *(idims + odims + self.ksize + self.stride + self.pad
-              + (coeff, gx)))
+              + (gx,)))
 
+        if not is_pad_value_none:
+            gx /= functools.reduce(operator.mul, self.ksize)
         return gx,
 
     def backward(self, indexes, grad_outputs):
         return AveragePoolingND(
             self.ndim, self.ksize, self.stride, self.pad,
-            cover_all=False).apply(grad_outputs)
+            cover_all=False, pad_value=self.pad_value).apply(grad_outputs)
 
 
 def average_pooling_nd(x, ksize, stride=None, pad=0, pad_value=0):
diff --git a/chainer/functions/pooling/average_pooling_nd_kernel.py b/chainer/functions/pooling/average_pooling_nd_kernel.py
--- a/chainer/functions/pooling/average_pooling_nd_kernel.py
+++ b/chainer/functions/pooling/average_pooling_nd_kernel.py
@@ -31,12 +31,6 @@ def name(self):
         # avg_pool_{N}d_bwd
         return 'avg'
 
-    def in_params(self):
-        # 2D: raw T gy, int32 d_0, int32 d_1, int32 out_0, int32 out_1,
-        #     int32 k_0, int32 k_1, int32 s_0, int32 s_1, int32 p_0,
-        #     int32 p_1, T coeff
-        return ['T coeff']
-
     def before(self):
         return 'T val = 0;'
 
@@ -45,4 +39,4 @@ def main(self, offset, xs, out_xs):
         return 'val = val + gy[{}];'.format(offset)
 
     def after(self, xs):
-        return 'gx = val * coeff;'
+        return 'gx = val;'


Test patch:
diff --git a/tests/chainer_tests/functions_tests/pooling_tests/test_average_pooling_nd.py b/tests/chainer_tests/functions_tests/pooling_tests/test_average_pooling_nd.py
--- a/tests/chainer_tests/functions_tests/pooling_tests/test_average_pooling_nd.py
+++ b/tests/chainer_tests/functions_tests/pooling_tests/test_average_pooling_nd.py
@@ -3,16 +3,14 @@
 import functools
 import numpy
 import operator
+import pytest
 import six
 
 import chainer
-from chainer import backend
 from chainer.backends import cuda
 from chainer import functions
-from chainer import gradient_check
 from chainer import testing
 from chainer.testing import attr
-from chainer.testing import condition
 from chainer.utils import conv
 from chainer_tests.functions_tests.pooling_tests import pooling_nd_helper
 
@@ -21,43 +19,58 @@
     'dims': [(4,), (4, 3), (4, 3, 2), (1, 1, 1, 1)],
     'dtype': [numpy.float16, numpy.float32, numpy.float64],
     'pad_value': [None, 0],
+    'contiguous': ['C', None],
 }))
-class TestAveragePoolingND(unittest.TestCase):
+@testing.inject_backend_tests(
+    None,
+    # CPU tests
+    [{}]
+    # GPU tests
+    + testing.product({
+        'use_cuda': [True],
+        'use_cudnn': ['never', 'always'],
+    })
+    # ChainerX tests
+    + [
+        {'use_chainerx': True, 'chainerx_device': 'native:0'},
+        {'use_chainerx': True, 'chainerx_device': 'cuda:0'},
+    ]
+)
+class TestAveragePoolingND(testing.FunctionTestCase):
 
     def setUp(self):
         self.ndim = len(self.dims)
         self.ksize = (3,) * self.ndim
         self.stride = (2,) * self.ndim
         self.pad = (1,) * self.ndim
-
-        x_shape = (2, 3) + self.dims
-        self.x = numpy.random.uniform(-1, 1, x_shape).astype(self.dtype)
-
+        self.input_shape = (2, 3) + self.dims
         outs = tuple(conv.get_conv_outsize(d, k, s, p, False)
                      for (d, k, s, p) in six.moves.zip(
                          self.dims, self.ksize, self.stride, self.pad))
-        gy_shape = (2, 3) + outs
-        self.gy = numpy.random.uniform(-1, 1, gy_shape).astype(self.dtype)
-        self.ggx = numpy.random.uniform(-1, 1, x_shape).astype(self.dtype)
+        self.output_shape = (2, 3) + outs
 
-        self.check_forward_options = {}
-        self.check_backward_options = {'eps': 1e-2}
+        self.check_backward_options.update({'atol': 5e-3, 'rtol': 5e-3})
+        self.check_double_backward_options.update({'atol': 5e-3, 'rtol': 5e-3})
         if self.dtype == numpy.float16:
-            self.check_forward_options = {'atol': 5e-4, 'rtol': 5e-3}
-            self.check_backward_options = {
-                'eps': 1e-2, 'atol': 5e-3, 'rtol': 5e-2}
+            self.check_forward_options.update({'atol': 5e-4, 'rtol': 5e-3})
+            self.check_backward_options.update({
+                'eps': 1e-2, 'atol': 5e-3, 'rtol': 5e-2})
+            self.check_backward_options.update({
+                'eps': 1e-2, 'atol': 5e-3, 'rtol': 5e-2})
+
+    def generate_inputs(self):
+        return numpy.random.uniform(
+            -1, 1, self.input_shape).astype(self.dtype),
+
+    def forward(self, inputs, device):
+        x, = inputs
+        return functions.average_pooling_nd(
+            x, self.ksize, self.stride, self.pad, self.pad_value),
 
-    def check_forward(self, x_data, use_cudnn='always'):
-        dims = self.dims
-        ksize = self.ksize
-        stride = self.stride
-        pad = self.pad
-        x = chainer.Variable(x_data)
-        with chainer.using_config('use_cudnn', use_cudnn):
-            y = functions.average_pooling_nd(
-                x, ksize, stride, pad, self.pad_value)
-        self.assertEqual(y.data.dtype, self.dtype)
-        y_data = cuda.to_cpu(y.data)
+    def forward_expected(self, inputs):
+        x, = inputs
+        patches = pooling_nd_helper.pooling_patches(
+            self.dims, self.ksize, self.stride, self.pad, False)
 
         def denom(idx):
             if self.pad_value is None:
@@ -66,213 +79,109 @@ def denom(idx):
                     s *= slic.stop - slic.start
                 return s
             else:
-                return functools.reduce(operator.mul, ksize)
+                return functools.reduce(operator.mul, self.ksize)
 
-        self.assertEqual(self.gy.shape, y_data.shape)
-        patches = pooling_nd_helper.pooling_patches(
-            dims, ksize, stride, pad, False)
+        y = []
         for k in six.moves.range(2):
+            tmp = []
             for c in six.moves.range(3):
-                x = self.x[k, c]
+                x_ = x[k, c]
                 expect = numpy.array(
-                    [x[idx].sum() / denom(idx) for idx in patches])
-                expect = expect.reshape(y_data.shape[2:])
-                testing.assert_allclose(
-                    expect, y_data[k, c], **self.check_forward_options)
-
-    def test_forward_cpu(self):
-        self.check_forward(self.x)
-
-    @attr.cudnn
-    def test_forward_gpu(self):
-        self.check_forward(cuda.to_gpu(self.x))
-
-    @attr.cudnn
-    def test_forward_gpu_non_contiguous(self):
-        self.check_forward(cuda.cupy.asfortranarray(cuda.to_gpu(self.x)))
-
-    @attr.gpu
-    def test_forward_gpu_no_cudnn(self):
-        self.check_forward(cuda.to_gpu(self.x), 'never')
+                    [x_[idx].sum() / denom(idx) for idx in patches])
+                expect = expect.reshape(self.output_shape[2:])
+                tmp.append(expect)
+            y.append(tmp)
+        return numpy.asarray(y, dtype=self.dtype),
 
-    @attr.chainerx
-    def test_forward_chainerx_native(self):
-        self.check_forward(backend.to_chx(self.x), 'never')
 
-    @attr.chainerx
-    @attr.gpu
-    def test_forward_chainerx_cuda(self):
-        self.check_forward(backend.to_chx(cuda.to_gpu(self.x)), 'never')
+@testing.parameterize(*testing.product({
+    'dtype': [numpy.float16, numpy.float32, numpy.float64],
+}))
+@testing.inject_backend_tests(
+    ['test_forward_consistency', 'test_backward_consistency'],
+    # CPU tests
+    [{}]
+    # GPU tests
+    + testing.product({
+        'use_cuda': [True],
+        'use_cudnn': ['never', 'always'],
+    })
+    # ChainerX tests
+    + testing.product({
+        'use_chainerx': [True],
+        'chainerx_device': ['native:0', 'cuda:0'],
+    })
+)
+class TestConsistencyAveragePoolingND(unittest.TestCase):
 
-    def check_forward_consistency_regression(self, x_data, use_cudnn='always'):
-        # Regression test to average_pooling_2d.
+    def setUp(self):
+        x_shape = (2, 3, 4, 3)
+        self.ksize = (3, 3)
+        self.stride = (2, 2)
+        self.pad = (1, 1)
+        self.pad_value = 0
+        self.x = numpy.random.uniform(-1, 1, x_shape).astype(self.dtype)
 
-        if len(self.dims) != 2:
-            return
+        outs = tuple(conv.get_conv_outsize(d, k, s, p, False)
+                     for (d, k, s, p) in six.moves.zip(
+                         x_shape[2:], self.ksize, self.stride, self.pad))
+        gy_shape = (2, 3) + outs
+        self.gy = numpy.random.uniform(-1, 1, gy_shape).astype(self.dtype)
 
-        if self.pad_value != 0:
-            # Not supported in average_pooling_2d
-            return
+        self.tolerance = {}
+        if self.dtype == numpy.float16:
+            self.tolerance.update({'atol': 1e-3, 'rtol': 1e-4})
 
+    def check_forward_consistency_regression(self, x_data, backend_config):
         ksize = self.ksize
         stride = self.stride
         pad = self.pad
+        pad_value = self.pad_value
 
-        with chainer.using_config('use_cudnn', use_cudnn):
+        with backend_config:
             y_nd = functions.average_pooling_nd(
-                x_data, ksize, stride=stride, pad=pad,
-                pad_value=self.pad_value)
+                x_data, ksize, stride=stride, pad=pad, pad_value=pad_value)
             y_2d = functions.average_pooling_2d(
                 x_data, ksize, stride=stride, pad=pad)
-        testing.assert_allclose(y_nd.data, y_2d.data)
-
-    def test_forward_consistency_regression_cpu(self):
-        self.check_forward_consistency_regression(self.x)
-
-    @attr.cudnn
-    def test_forward_consistency_regression_gpu(self):
-        self.check_forward_consistency_regression(cuda.to_gpu(self.x))
-
-    @attr.gpu
-    def test_forward_consistency_regression_no_cudnn(self):
-        self.check_forward_consistency_regression(cuda.to_gpu(self.x), 'never')
-
-    def check_backward(self, x_data, y_grad, use_cudnn='always'):
-        def f(x):
-            return functions.average_pooling_nd(
-                x, self.ksize, stride=self.stride, pad=self.pad)
-
-        with chainer.using_config('use_cudnn', use_cudnn):
-            gradient_check.check_backward(
-                f, x_data, y_grad, dtype=numpy.float64,
-                **self.check_backward_options)
-
-    def test_backward_cpu(self):
-        self.check_backward(self.x, self.gy)
 
-    @attr.cudnn
-    def test_backward_gpu(self):
-        self.check_backward(cuda.to_gpu(self.x), cuda.to_gpu(self.gy))
+        testing.assert_allclose(y_nd.array, y_2d.array, **self.tolerance)
 
-    @attr.cudnn
-    def test_backward_gpu_non_contiguous(self):
-        self.check_backward(
-            cuda.cupy.asfortranarray(cuda.to_gpu(self.x)),
-            cuda.cupy.asfortranarray(cuda.to_gpu(self.gy)))
+    def test_forward_consistency(self, backend_config):
+        x = self.x.copy()
+        x = backend_config.get_array(x)
+        self.check_forward_consistency_regression(x, backend_config)
 
-    @attr.gpu
-    def test_backward_gpu_no_cudnn(self):
-        self.check_backward(cuda.to_gpu(self.x), cuda.to_gpu(self.gy), 'never')
-
-    @attr.chainerx
-    def test_backward_chainerx_native(self):
-        self.check_backward(
-            backend.to_chx(self.x), backend.to_chx(self.gy), 'never')
-
-    @attr.chainerx
-    @attr.gpu
-    def test_backward_chainerx_cuda(self):
-        def conv(a):
-            return backend.to_chx(cuda.to_gpu(a))
-
-        self.check_backward(conv(self.x), conv(self.gy), 'never')
-
-    def check_backward_consistency_regression(self, x_data, gy_data,
-                                              use_cudnn='always'):
+    def check_backward_consistency_regression(
+            self, x_data, gy_data, backend_config):
         # Regression test to two-dimensional average pooling layer.
-
-        if len(self.dims) != 2:
-            return
-
         ksize = self.ksize
         stride = self.stride
         pad = self.pad
-        xp = backend.get_array_module(x_data)
+        pad_value = self.pad_value
 
         # Backward computation for N-dimensional average pooling layer.
-        x_nd = chainer.Variable(xp.array(x_data))
-        with chainer.using_config('use_cudnn', use_cudnn):
+        x_nd = chainer.Variable(x_data)
+        with backend_config:
             y_nd = functions.average_pooling_nd(
-                x_nd, ksize, stride=stride, pad=pad)
-
+                x_nd, ksize, stride=stride, pad=pad, pad_value=pad_value)
         y_nd.grad = gy_data
         y_nd.backward()
 
         # Backward computation for two-dimensional average pooling layer.
-        x_2d = chainer.Variable(xp.array(x_data))
-        with chainer.using_config('use_cudnn', use_cudnn):
+        x_2d = chainer.Variable(x_data)
+        with backend_config:
             y_2d = functions.average_pooling_2d(
                 x_2d, ksize, stride=stride, pad=pad)
-
         y_2d.grad = gy_data
         y_2d.backward()
 
         # Test that the two result gradients are close enough.
-        testing.assert_allclose(x_nd.grad, x_2d.grad)
-
-    def test_backward_consistency_regression_cpu(self):
-        self.check_backward_consistency_regression(self.x, self.gy)
-
-    @attr.cudnn
-    def test_backward_consistency_regression_gpu(self):
-        self.check_backward_consistency_regression(
-            cuda.to_gpu(self.x), cuda.to_gpu(self.gy))
-
-    @attr.gpu
-    def test_backward_consistency_regression_no_cudnn(self):
-        self.check_backward_consistency_regression(
-            cuda.to_gpu(self.x), cuda.to_gpu(self.gy), use_cudnn='never')
-
-    def check_double_backward(self, x_data, y_grad, x_grad_grad,
-                              use_cudnn='always'):
-        def f(x):
-            return functions.average_pooling_nd(
-                x, self.ksize, stride=self.stride, pad=self.pad)
-        with chainer.using_config('use_cudnn', use_cudnn):
-            gradient_check.check_double_backward(
-                f, x_data, y_grad, x_grad_grad, **self.check_backward_options)
-
-    @condition.retry(10)
-    def test_double_backward_cpu(self):
-        self.check_double_backward(self.x, self.gy, self.ggx, 'never')
-
-    @attr.cudnn
-    @condition.retry(10)
-    def test_double_backward_gpu(self):
-        self.check_double_backward(
-            cuda.to_gpu(self.x), cuda.to_gpu(self.gy), cuda.to_gpu(self.ggx))
-
-    @attr.cudnn
-    @condition.retry(10)
-    def test_double_backward_gpu_non_contiguous(self):
-        self.check_double_backward(
-            cuda.cupy.asfortranarray(cuda.to_gpu(self.x)),
-            cuda.cupy.asfortranarray(cuda.to_gpu(self.gy)),
-            cuda.cupy.asfortranarray(cuda.to_gpu(self.ggx)))
-
-    @attr.gpu
-    @condition.retry(10)
-    def test_double_backward_gpu_no_cudnn(self):
-        self.check_double_backward(
-            cuda.to_gpu(self.x), cuda.to_gpu(self.gy), cuda.to_gpu(self.ggx),
-            'never')
-
-    @attr.chainerx
-    @condition.retry(10)
-    def test_double_backward_chainerx_native(self):
-        self.check_double_backward(
-            backend.to_chx(self.x), backend.to_chx(self.gy),
-            backend.to_chx(self.ggx), 'never')
-
-    @attr.chainerx
-    @attr.gpu
-    @condition.retry(10)
-    def test_double_backward_chainerx_cuda(self):
-        def conv(a):
-            return backend.to_chx(cuda.to_gpu(a))
-
-        self.check_double_backward(
-            conv(self.x), conv(self.gy), conv(self.ggx), 'never')
+        testing.assert_allclose(x_nd.grad, x_2d.grad, **self.tolerance)
+
+    def test_backward_consistency(self, backend_config):
+        x = backend_config.get_array(self.x)
+        gy = backend_config.get_array(self.gy)
+        self.check_backward_consistency_regression(x, gy, backend_config)
 
 
 @testing.parameterize(*testing.product({
@@ -306,9 +215,8 @@ def test_call_cudnn_forward(self):
         with chainer.using_config('use_cudnn', self.use_cudnn):
             with testing.patch('cupy.cudnn.pooling_forward') as func:
                 self.forward()
-                self.assertEqual(func.called,
-                                 chainer.should_use_cudnn('>=auto') and
-                                 self.ndim > 1)
+                assert func.called == (
+                    chainer.should_use_cudnn('>=auto') and self.ndim > 1)
 
     def test_call_cudnn_backward(self):
         with chainer.using_config('use_cudnn', self.use_cudnn):
@@ -318,7 +226,7 @@ def test_call_cudnn_backward(self):
         y.grad = self.gy
         with testing.patch('cupy.cudnn.pooling_backward') as func:
             y.backward()
-            self.assertEqual(func.called, expect)
+            assert func.called == expect
 
 
 class TestAveragePoolingNDWrappers(unittest.TestCase):
@@ -335,12 +243,12 @@ def _get_data(self, ndim):
     def test_average_pooling_1d(self):
         (x, ksize) = self._get_data(1)
         testing.assert_allclose(
-            functions.average_pooling_nd(x, ksize).data,
-            functions.average_pooling_1d(x, ksize).data)
+            functions.average_pooling_nd(x, ksize).array,
+            functions.average_pooling_1d(x, ksize).array)
 
     def test_average_pooling_1d_invalid(self):
         (x, ksize) = self._get_data(2)
-        with self.assertRaises(ValueError):
+        with pytest.raises(ValueError):
             functions.average_pooling_1d(x, ksize)
 
     def test_average_pooling_3d(self):
@@ -351,7 +259,7 @@ def test_average_pooling_3d(self):
 
     def test_average_pooling_3d_invalid(self):
         (x, ksize) = self._get_data(2)
-        with self.assertRaises(ValueError):
+        with pytest.raises(ValueError):
             functions.average_pooling_3d(x, ksize)