Repository: chainer/chainer

Issue:
deconvolution_2d with non c-contiguous array
Current deconvolution_2d doesn't support backward with non c-contiguous array.
Should we put ascontinousarray() in appropriate places ?

```
if not gy.flags.c_contiguous:
      gy = cuda.cupy.ascontiguousarray(gy)
```



Discussion:

Resolving patch:
diff --git a/chainer/functions/connection/convolution_2d.py b/chainer/functions/connection/convolution_2d.py
--- a/chainer/functions/connection/convolution_2d.py
+++ b/chainer/functions/connection/convolution_2d.py
@@ -60,16 +60,21 @@ def forward_cpu(self, inputs):
 
     def forward_gpu(self, inputs):
         x, W = inputs[:2]
-        out_c, _, kh, kw = W.shape
+        b = inputs[2] if len(inputs) == 3 else None
 
+        out_c, _, kh, kw = W.shape
         n, c, h, w = x.shape
-        b = inputs[2] if len(inputs) == 3 else None
 
         out_h = conv.get_conv_outsize(h, kh, self.sy, self.ph)
         out_w = conv.get_conv_outsize(w, kw, self.sx, self.pw)
 
         y = cuda.cupy.empty((n, out_c, out_h, out_w), dtype=x.dtype)
         if cuda.cudnn_enabled and self.use_cudnn:
+            x = cuda.cupy.ascontiguousarray(x)
+            W = cuda.cupy.ascontiguousarray(W)
+            if b is not None:
+                b = cuda.cupy.ascontiguousarray(b)
+
             handle = cudnn.get_handle()
             x_desc = cudnn.create_tensor_descriptor(x)
             y_desc = cudnn.create_tensor_descriptor(y)
@@ -125,6 +130,7 @@ def forward_gpu(self, inputs):
 
     def backward_cpu(self, inputs, grad_outputs):
         x, W = inputs[:2]
+        b = inputs[2] if len(inputs) == 3 else None
         gy = grad_outputs[0]
         h, w = x.shape[2:]
 
@@ -133,11 +139,11 @@ def backward_cpu(self, inputs, grad_outputs):
         gcol = numpy.rollaxis(gcol, 3)
         gx = conv.col2im_cpu(gcol, self.sy, self.sx, self.ph, self.pw, h, w)
 
-        if len(inputs) == 3:
+        if b is None:
+            return gx, gW
+        else:
             gb = gy.sum(axis=(0, 2, 3))
             return gx, gW, gb
-        else:
-            return gx, gW
 
     def backward_gpu(self, inputs, grad_outputs):
         x, W = inputs[:2]
@@ -149,10 +155,12 @@ def backward_gpu(self, inputs, grad_outputs):
 
         gW = cuda.cupy.empty_like(W)
         if cuda.cudnn_enabled and self.use_cudnn:
+            x = cuda.cupy.ascontiguousarray(x)
+            W = cuda.cupy.ascontiguousarray(W)
+            gy = cuda.cupy.ascontiguousarray(gy)
+
             handle = cudnn.get_handle()
             x_desc = cudnn.create_tensor_descriptor(x)
-            if not gy.flags.c_contiguous:
-                gy = cuda.cupy.ascontiguousarray(gy)
             gy_desc = cudnn.create_tensor_descriptor(gy)
             dtype = x.dtype
             one = numpy.array(1, dtype=dtype).ctypes
@@ -170,7 +178,7 @@ def backward_gpu(self, inputs, grad_outputs):
                 zero.data, x_desc.value, gx.data.ptr)
 
             if b is not None:
-                gb = cuda.cupy.empty_like(inputs[2])
+                gb = cuda.cupy.empty_like(b)
                 libcudnn.convolutionBackwardBias(
                     handle, one.data, gy_desc.value, gy.data.ptr,
                     zero.data, self.bias_desc.value, gb.data.ptr)
diff --git a/chainer/functions/connection/deconvolution_2d.py b/chainer/functions/connection/deconvolution_2d.py
--- a/chainer/functions/connection/deconvolution_2d.py
+++ b/chainer/functions/connection/deconvolution_2d.py
@@ -62,6 +62,7 @@ def check_type_forward(self, in_types):
 
     def forward_cpu(self, inputs):
         x, W = inputs[:2]
+        b = inputs[2] if len(inputs) == 3 else None
         kh, kw = W.shape[2:]
         _, _, h, w = x.shape
         gcol = numpy.tensordot(W, x, (0, 1))
@@ -77,13 +78,13 @@ def forward_cpu(self, inputs):
         y = conv.col2im_cpu(
             gcol, self.sy, self.sx, self.ph, self.pw, self.outh, self.outw)
         # b, k, h, w
-        if len(inputs) == 3:
-            b = inputs[2]
+        if b is not None:
             y += b.reshape(1, b.size, 1, 1)
         return y,
 
     def forward_gpu(self, inputs):
         x, W = inputs[:2]
+        b = inputs[2] if len(inputs) == 3 else None
         kh, kw = W.shape[2:]
         n, in_c, in_h, in_w = x.shape
         c = W.shape[1]  # out_c
@@ -91,9 +92,12 @@ def forward_gpu(self, inputs):
             self.outh = conv.get_deconv_outsize(in_h, kh, self.sy, self.ph)
         if self.outw is None:
             self.outw = conv.get_deconv_outsize(in_w, kw, self.sx, self.pw)
-        if len(inputs) == 3:
-            b = inputs[2]
         if cuda.cudnn_enabled and self.use_cudnn:
+            x = cuda.cupy.ascontiguousarray(x)
+            W = cuda.cupy.ascontiguousarray(W)
+            if b is not None:
+                b = cuda.cupy.ascontiguousarray(b)
+
             handle = cudnn.get_handle()
             x_desc = cudnn.create_tensor_descriptor(x)
             y = cuda.cupy.empty((n, c, self.outh, self.outw),
@@ -103,7 +107,7 @@ def forward_gpu(self, inputs):
             self.filter_desc = cudnn.create_filter_descriptor(W)
             self.conv_desc = cudnn.create_convolution_descriptor(
                 (self.ph, self.pw), (self.sy, self.sx))
-            if len(inputs) == 3:
+            if b is not None:
                 self.bias_desc = cudnn.create_tensor_descriptor(
                     b[None, :, None, None])
 
@@ -114,7 +118,7 @@ def forward_gpu(self, inputs):
                 handle, one.data, self.filter_desc.value, W.data.ptr,
                 x_desc.value, x.data.ptr, self.conv_desc.value,
                 zero.data, y_desc.value, y.data.ptr)
-            if len(inputs) == 3:
+            if b is not None:
                 libcudnn.addTensor_v2(
                     handle, libcudnn.CUDNN_ADD_SAME_C,
                     one.data, self.bias_desc.value, b.data.ptr,
@@ -129,12 +133,13 @@ def forward_gpu(self, inputs):
                 cuda.cupy.dot(W_mat.T, x_mats[i], gcol_mats[i])
             y = conv.col2im_gpu(
                 gcol, self.sy, self.sx, self.ph, self.pw, self.outh, self.outw)
-            if len(inputs) == 3:
+            if b is not None:
                 y += b.reshape(1, b.size, 1, 1)
         return y,
 
     def backward_cpu(self, inputs, grad_outputs):
         x, W = inputs[:2]
+        b = inputs[2] if len(inputs) == 3 else None
         gy = grad_outputs[0]
         kh, kw = W.shape[2:]
         col = conv.im2col_cpu(
@@ -143,14 +148,15 @@ def backward_cpu(self, inputs, grad_outputs):
         gx = numpy.tensordot(col, W, ([1, 2, 3], [1, 2, 3]))
         gx = numpy.rollaxis(gx, 3, 1)
 
-        if len(inputs) == 3:
+        if b is None:
+            return gx, gW
+        else:
             gb = gy.sum(axis=(0, 2, 3))
             return gx, gW, gb
-        else:
-            return gx, gW
 
     def backward_gpu(self, inputs, grad_outputs):
         x, W = inputs[:2]
+        b = inputs[2] if len(inputs) == 3 else None
         gy = grad_outputs[0]
         n, in_c, in_h, in_w = x.shape
         _, out_channels, kh, kw = W.shape
@@ -158,6 +164,10 @@ def backward_gpu(self, inputs, grad_outputs):
         gx = cuda.cupy.empty((n, in_c, in_h, in_w), dtype=numpy.float32)
 
         if cuda.cudnn_enabled and self.use_cudnn:
+            x = cuda.cupy.ascontiguousarray(x)
+            W = cuda.cupy.ascontiguousarray(W)
+            gy = cuda.cupy.ascontiguousarray(gy)
+
             handle = cudnn.get_handle()
             gy_desc = cudnn.create_tensor_descriptor(gy)
             gx_desc = cudnn.create_tensor_descriptor(gx)
@@ -183,8 +193,7 @@ def backward_gpu(self, inputs, grad_outputs):
                 self.conv_desc.value, algo, workspace.data.ptr, workspace_size,
                 zero.data, gx_desc.value, gx.data.ptr)
             # bias backward
-            if len(inputs) == 3:
-                b = inputs[2]
+            if b is not None:
                 gb = cuda.cupy.empty_like(b)
                 libcudnn.convolutionBackwardBias(
                     handle, one.data, gy_desc.value, gy.data.ptr,
@@ -208,7 +217,7 @@ def backward_gpu(self, inputs, grad_outputs):
                 gx_mats[i] = W_mat.dot(col_mats[i])
 
             # bias backward
-            if len(inputs) == 3:
+            if b is not None:
                 gb = gy.sum(axis=(0, 2, 3))
 
             # filter backward
@@ -217,10 +226,11 @@ def backward_gpu(self, inputs, grad_outputs):
             x_mats = x.reshape(n, in_c, in_h * in_w)
             for i in moves.range(n):
                 gW_mat += x_mats[i].dot(col_mats[i].T)
-        if len(inputs) == 3:
-            return gx, gW, gb
-        else:
+
+        if b is None:
             return gx, gW
+        else:
+            return gx, gW, gb
 
 
 def deconvolution_2d(x, W, b=None, stride=1, pad=0,
diff --git a/chainer/gradient_check.py b/chainer/gradient_check.py
--- a/chainer/gradient_check.py
+++ b/chainer/gradient_check.py
@@ -1,5 +1,4 @@
 import numpy
-import six
 
 from chainer import cuda
 from chainer import utils
@@ -11,54 +10,6 @@ def _copy_arrays(xs):
     return tuple(xp.copy(x) for x in xs)
 
 
-def numerical_grad_cpu(f, inputs, grad_outputs, eps=1e-3):
-    grads = tuple(numpy.zeros_like(x) for x in inputs)
-    for x, gx in zip(inputs, grads):
-        flat_x = x.ravel()
-        flat_gx = gx.ravel()
-        for i in six.moves.range(flat_x.size):
-            orig = flat_x[i]
-            flat_x[i] = orig + eps
-            ys1 = _copy_arrays(f())
-            flat_x[i] = orig - eps
-            ys2 = _copy_arrays(f())
-            flat_x[i] = orig
-
-            for y1, y2, gy in zip(ys1, ys2, grad_outputs):
-                if gy is not None:
-                    dot = float(sum(((y1 - y2) * gy).ravel()))
-                    flat_gx[i] += dot / (2 * eps)
-
-    return grads
-
-
-def numerical_grad_gpu(f, inputs, grad_outputs, eps=1e-3):
-    grads = tuple(cuda.cupy.zeros_like(x) for x in inputs)
-    for x, gx in zip(inputs, grads):
-        x = x.ravel()
-        gx = gx.ravel()
-        x_cpu = x.get()
-        gx_cpu = gx.get()
-        for i in six.moves.range(x_cpu.size):
-            orig = x_cpu[i]
-            x_cpu[i] = orig + eps
-            x.set(x_cpu)
-            ys1 = _copy_arrays(f())
-            x_cpu[i] = orig - eps
-            x.set(x_cpu)
-            ys2 = _copy_arrays(f())
-            x_cpu[i] = orig
-            x.set(x_cpu)
-
-            for y1, y2, gy in zip(ys1, ys2, grad_outputs):
-                if gy is not None:
-                    dot = sum(((y1 - y2) * gy).ravel()).get()
-                    gx_cpu[i] += dot / (2 * eps)
-        gx.set(gx_cpu)
-
-    return grads
-
-
 def numerical_grad(f, inputs, grad_outputs, eps=1e-3):
     """Computes numerical gradient by finite differences.
 
@@ -83,15 +34,29 @@ def numerical_grad(f, inputs, grad_outputs, eps=1e-3):
     inputs = tuple(inputs)
     grad_outputs = tuple(grad_outputs)
     gpu = any(isinstance(x, cuda.ndarray) for x in inputs + grad_outputs)
-
     cpu = any(isinstance(x, numpy.ndarray) for x in inputs + grad_outputs)
 
     if gpu and cpu:
         raise RuntimeError('Do not mix GPU and CPU arrays in `numerical_grad`')
-    elif gpu:
-        return numerical_grad_gpu(f, inputs, grad_outputs, eps)
+
+    if gpu:
+        xp = cuda.cupy
     else:
-        return numerical_grad_cpu(f, inputs, grad_outputs, eps)
+        xp = numpy
+    grads = tuple(xp.zeros_like(x) for x in inputs)
+    for x, gx in zip(inputs, grads):
+        for i in numpy.ndindex(x.shape):
+            orig = x[i].copy()  # hold original value
+            x[i] = orig + eps
+            ys1 = _copy_arrays(f())
+            x[i] = orig - eps
+            ys2 = _copy_arrays(f())
+            x[i] = orig
+            for y1, y2, gy in zip(ys1, ys2, grad_outputs):
+                if gy is not None:
+                    dot = ((y1 - y2) * gy).sum()
+                    gx[i] += dot / (2 * eps)
+    return grads
 
 
 def assert_allclose(x, y, atol=1e-5, rtol=1e-4, verbose=True):


Test patch:
diff --git a/tests/chainer_tests/functions_tests/connection_tests/test_convolution_2d.py b/tests/chainer_tests/functions_tests/connection_tests/test_convolution_2d.py
--- a/tests/chainer_tests/functions_tests/connection_tests/test_convolution_2d.py
+++ b/tests/chainer_tests/functions_tests/connection_tests/test_convolution_2d.py
@@ -12,6 +12,17 @@
 from chainer.testing import condition
 
 
+def _asfortranarray(x):
+    xp = cuda.get_array_module(x)
+    if xp is numpy:
+        return xp.asfortranarray(x)
+    else:
+        return xp.ascontiguousarray(x.T).T
+
+
+@testing.parameterize(*testing.product({
+    'c_contiguous': [True, False],
+}))
 class TestConvolution2DFunction(unittest.TestCase):
 
     def setUp(self, use_cudnn=True):
@@ -61,6 +72,20 @@ def test_forward_consistency_im2col_nobias(self):
         self.test_forward_consistency(nobias=True)
 
     def check_backward(self, x_data, W_data, b_data, y_grad):
+        if not self.c_contiguous:
+            x_data = _asfortranarray(x_data)
+            W_data = _asfortranarray(W_data)
+            y_grad = _asfortranarray(y_grad)
+            self.assertFalse(x_data.flags.c_contiguous)
+            self.assertFalse(W_data.flags.c_contiguous)
+            self.assertFalse(y_grad.flags.c_contiguous)
+            if b_data is not None:
+                xp = cuda.get_array_module(b_data)
+                b = xp.empty((len(b_data) * 2,), dtype=self.b.dtype)
+                b[::2] = b_data
+                b_data = b[::2]
+                self.assertFalse(b_data.flags.c_contiguous)
+
         args = (x_data, W_data)
         if b_data is not None:
             args = args + (b_data,)
diff --git a/tests/chainer_tests/functions_tests/connection_tests/test_deconvolution_2d.py b/tests/chainer_tests/functions_tests/connection_tests/test_deconvolution_2d.py
--- a/tests/chainer_tests/functions_tests/connection_tests/test_deconvolution_2d.py
+++ b/tests/chainer_tests/functions_tests/connection_tests/test_deconvolution_2d.py
@@ -20,6 +20,14 @@ def _pair(x):
     return (x, x)
 
 
+def _asfortranarray(x):
+    xp = cuda.get_array_module(x)
+    if xp is numpy:
+        return xp.asfortranarray(x)
+    else:
+        return xp.ascontiguousarray(x.T).T
+
+
 @parameterize(
     *testing.product({
         'in_channels': [3],
@@ -31,6 +39,7 @@ def _pair(x):
         'nobias': [True, False],
         'use_cudnn': [True, False],
         'test_outsize': [True, False],
+        'c_contiguous': [True, False],
     })
 )
 class TestDeconvolution2DFunction(unittest.TestCase):
@@ -80,6 +89,20 @@ def test_forward_consistency_im2col(self):
         self.test_forward_consistency()
 
     def check_backward(self, x_data, W_data, b_data, y_grad):
+        if not self.c_contiguous:
+            x_data = _asfortranarray(x_data)
+            W_data = _asfortranarray(W_data)
+            y_grad = _asfortranarray(y_grad)
+            self.assertFalse(x_data.flags.c_contiguous)
+            self.assertFalse(W_data.flags.c_contiguous)
+            self.assertFalse(y_grad.flags.c_contiguous)
+            if b_data is not None:
+                xp = cuda.get_array_module(b_data)
+                b = xp.empty((len(b_data) * 2,), dtype=self.b.dtype)
+                b[::2] = b_data
+                b_data = b[::2]
+                self.assertFalse(b_data.flags.c_contiguous)
+
         args = (x_data, W_data)
         if b_data is not None:
             args = args + (b_data,)