Repository: chainer/chainer

Issue:
Convolution2D does not support 64 bit
When I change dtype=numpy.float32 to dtype=numpy.float64 in the constructor of Convolution2D, and then run nosetests I get tons of errors. I think 64 bit is simply not supported, so we should raise a NotImplementedError.



Discussion:
You mean you changed the default argument of `dtype` in `Convolution2D.__init__` from `numpy.float32` to `numpy.float64` ? Some unit tests assume the default value of `dtype` be `float32` (e.g. test_forward_consistency). So I think we cannot say that passing `float64` as the argument of `__init__` does not work only from the test errors (although we cannot say it does work, neither because we do not have test cases where `dtype` equals to `float64`).

Yes, that's what I did. You're right that this does not exclude other causes for the error, so I tried the following:

```
import cupy as cp
from chainer.functions.connection.convolution_2d import Convolution2D
S32 = Convolution2D(in_channels=1, out_channels=2, ksize=5, dtype='float32', use_cudnn=True)
S64 = Convolution2D(in_channels=1, out_channels=2, ksize=5, dtype='float64', use_cudnn=True)

S32.to_gpu()
S64.to_gpu()

x_data32 = cp.random.randn(3, 1, 7, 7).astype('float32')
x32 = Variable(x_data32)
x64 = Variable(x_data32.astype('float64'))

S32.W = cp.random.randn(*S32.W.shape).astype('float32')

S64.W = S32.W.astype('float64')
S64.b = S32.b.astype('float64')

y32 = S32(x32)
y64 = S64(x64)

print 'difference', cp.abs(y32.data - y64.data).sum()
```

The output y64 is all zero. It works fine when I change use_cudnn to False for the 64 bit conv, so it seems like this is a problem with cudnn.

I looked at the Convolution2D code but could not find the error. One thing I did notice is that the workspace_size is multiplied by 4 (number of bytes in a float). I changed this 4 to self.dtype.itemsize, and also changed the line
   workspace = cuda.empty((max(workspace_size // 4, 1),), dtype=self.dtype)
in forward_gpu to divide by self.dtype.itemsize. Still results in zero output.

I'm not familiar with cudnn, so I don't know what could be causing this. For now I'm going to stick with the non-cudnn 64 bit version for my gradient checking.


Resolving patch:
diff --git a/chainer/functions/connection/convolution_2d.py b/chainer/functions/connection/convolution_2d.py
--- a/chainer/functions/connection/convolution_2d.py
+++ b/chainer/functions/connection/convolution_2d.py
@@ -18,6 +18,11 @@
             libcudnn.CUDNN_CONVOLUTION_BWD_DATA_SPECIFY_WORKSPACE_LIMIT
 
 
+def _check_cudnn_acceptable_type(x_dtype, W_dtype):
+    return x_dtype == W_dtype and (
+        _cudnn_version >= 3000 or x_dtype != numpy.float16)
+
+
 def _pair(x):
     if hasattr(x, '__getitem__'):
         return x
@@ -39,8 +44,8 @@ def check_type_forward(self, in_types):
         x_type = in_types[0]
         w_type = in_types[1]
         type_check.expect(
-            x_type.dtype == numpy.float32,
-            w_type.dtype == numpy.float32,
+            x_type.dtype.kind == 'f',
+            w_type.dtype.kind == 'f',
             x_type.ndim == 4,
             w_type.ndim == 4,
             x_type.shape[1] == w_type.shape[1],
@@ -49,7 +54,7 @@ def check_type_forward(self, in_types):
         if n_in.eval() == 3:
             b_type = in_types[2]
             type_check.expect(
-                b_type.dtype == numpy.float32,
+                b_type.dtype == x_type.dtype,
                 b_type.ndim == 1,
                 b_type.shape[0] == w_type.shape[0],
             )
@@ -61,7 +66,8 @@ def forward_cpu(self, inputs):
         self.col = conv.im2col_cpu(
             x, kh, kw, self.sy, self.sx, self.ph, self.pw,
             cover_all=self.cover_all)
-        y = numpy.tensordot(self.col, W, ((1, 2, 3), (1, 2, 3)))
+        y = numpy.tensordot(
+            self.col, W, ((1, 2, 3), (1, 2, 3))).astype(x.dtype)
         if b is not None:
             y += b
         return numpy.rollaxis(y, 3, 1),
@@ -79,7 +85,8 @@ def forward_gpu(self, inputs):
                                       cover_all=self.cover_all)
 
         y = cuda.cupy.empty((n, out_c, out_h, out_w), dtype=x.dtype)
-        if not self.cover_all and cuda.cudnn_enabled and self.use_cudnn:
+        if (not self.cover_all and cuda.cudnn_enabled and self.use_cudnn and
+                _check_cudnn_acceptable_type(x.dtype, W.dtype)):
             x = cuda.cupy.ascontiguousarray(x)
             W = cuda.cupy.ascontiguousarray(W)
             if b is not None:
@@ -103,9 +110,9 @@ def forward_gpu(self, inputs):
                 self.conv_desc.value, y_desc.value, _fwd_pref,
                 workspace_size)
 
-            dtype = x.dtype
-            one = numpy.array(1, dtype=dtype).ctypes
-            zero = numpy.array(0, dtype=dtype).ctypes
+            oz_dtype = 'd' if x.dtype == 'd' else 'f'
+            one = numpy.array(1, dtype=oz_dtype).ctypes
+            zero = numpy.array(0, dtype=oz_dtype).ctypes
             libcudnn.convolutionForward(
                 handle, one.data, x_desc.value, x.data.ptr,
                 self.filter_desc.value, W.data.ptr, self.conv_desc.value,
@@ -140,8 +147,9 @@ def backward_cpu(self, inputs, grad_outputs):
         gy = grad_outputs[0]
         h, w = x.shape[2:]
 
-        gW = numpy.tensordot(gy, self.col, ((0, 2, 3), (0, 4, 5)))
-        gcol = numpy.tensordot(W, gy, (0, 1))
+        gW = numpy.tensordot(
+            gy, self.col, ((0, 2, 3), (0, 4, 5))).astype(W.dtype)
+        gcol = numpy.tensordot(W, gy, (0, 1)).astype(x.dtype)
         gcol = numpy.rollaxis(gcol, 3)
         gx = conv.col2im_cpu(gcol, self.sy, self.sx, self.ph, self.pw, h, w)
 
@@ -160,7 +168,8 @@ def backward_gpu(self, inputs, grad_outputs):
         kh, kw = W.shape[2:]
 
         gW = cuda.cupy.empty_like(W)
-        if not self.cover_all and cuda.cudnn_enabled and self.use_cudnn:
+        if (not self.cover_all and cuda.cudnn_enabled and self.use_cudnn and
+                _check_cudnn_acceptable_type(x.dtype, W.dtype)):
             x = cuda.cupy.ascontiguousarray(x)
             W = cuda.cupy.ascontiguousarray(W)
             gy = cuda.cupy.ascontiguousarray(gy)
@@ -168,9 +177,9 @@ def backward_gpu(self, inputs, grad_outputs):
             handle = cudnn.get_handle()
             x_desc = cudnn.create_tensor_descriptor(x)
             gy_desc = cudnn.create_tensor_descriptor(gy)
-            dtype = x.dtype
-            one = numpy.array(1, dtype=dtype).ctypes
-            zero = numpy.array(0, dtype=dtype).ctypes
+            oz_dtype = 'd' if x.dtype == 'd' else 'f'
+            one = numpy.array(1, dtype=oz_dtype).ctypes
+            zero = numpy.array(0, dtype=oz_dtype).ctypes
             gx = cuda.cupy.empty_like(x)
 
             if _cudnn_version >= 4000:
@@ -223,8 +232,9 @@ def backward_gpu(self, inputs, grad_outputs):
             W_mat = W.reshape(out_c, -1)
             gcol = cuda.cupy.empty_like(self.col)
             gcol_mats = gcol.reshape(n, c * kh * kw, out_h * out_w)
+
             for i in moves.range(n):
-                cuda.cupy.dot(W_mat.T, gy_mats[i], gcol_mats[i])
+                gcol_mats[i] = cuda.cupy.dot(W_mat.T, gy_mats[i])
 
             gx = conv.col2im_gpu(
                 gcol, self.sy, self.sx, self.ph, self.pw, h, w)
diff --git a/chainer/functions/connection/deconvolution_2d.py b/chainer/functions/connection/deconvolution_2d.py
--- a/chainer/functions/connection/deconvolution_2d.py
+++ b/chainer/functions/connection/deconvolution_2d.py
@@ -3,6 +3,7 @@
 
 from chainer import cuda
 from chainer import function
+from chainer.functions.connection import convolution_2d
 from chainer.utils import conv
 from chainer.utils import type_check
 
@@ -18,6 +19,9 @@
             libcudnn.CUDNN_CONVOLUTION_BWD_DATA_SPECIFY_WORKSPACE_LIMIT
 
 
+_check_cudnn_acceptable_type = convolution_2d._check_cudnn_acceptable_type
+
+
 def _pair(x):
     if hasattr(x, '__getitem__'):
         return x
@@ -38,8 +42,8 @@ def check_type_forward(self, in_types):
         x_type, w_type = in_types[:2]
 
         type_check.expect(
-            x_type.dtype == numpy.float32,
-            w_type.dtype == numpy.float32,
+            x_type.dtype.kind == 'f',
+            w_type.dtype.kind == 'f',
             x_type.ndim == 4,
             w_type.ndim == 4,
             x_type.shape[1] == w_type.shape[0]
@@ -61,7 +65,7 @@ def check_type_forward(self, in_types):
         if n_in.eval() == 3:
             b_type = in_types[2]
             type_check.expect(
-                b_type.dtype == numpy.float32,
+                b_type.dtype == x_type.dtype,
                 b_type.ndim == 1,
                 b_type.shape[0] == w_type.shape[1]
             )
@@ -71,7 +75,7 @@ def forward_cpu(self, inputs):
         b = inputs[2] if len(inputs) == 3 else None
         kh, kw = W.shape[2:]
         _, _, h, w = x.shape
-        gcol = numpy.tensordot(W, x, (0, 1))
+        gcol = numpy.tensordot(W, x, (0, 1)).astype(x.dtype)
         # - k, m, n: shape of out_channel
         # - b: number of inputs
         # - h, w: height and width of kernels
@@ -98,7 +102,8 @@ def forward_gpu(self, inputs):
             self.outh = conv.get_deconv_outsize(in_h, kh, self.sy, self.ph)
         if self.outw is None:
             self.outw = conv.get_deconv_outsize(in_w, kw, self.sx, self.pw)
-        if cuda.cudnn_enabled and self.use_cudnn:
+        if (cuda.cudnn_enabled and self.use_cudnn and
+                _check_cudnn_acceptable_type(x.dtype, W.dtype)):
             x = cuda.cupy.ascontiguousarray(x)
             W = cuda.cupy.ascontiguousarray(W)
             if b is not None:
@@ -107,7 +112,7 @@ def forward_gpu(self, inputs):
             handle = cudnn.get_handle()
             x_desc = cudnn.create_tensor_descriptor(x)
             y = cuda.cupy.empty((n, c, self.outh, self.outw),
-                                dtype=numpy.float32)
+                                dtype=x.dtype)
             y_desc = cudnn.create_tensor_descriptor(y)
 
             self.filter_desc = cudnn.create_filter_descriptor(W)
@@ -117,8 +122,9 @@ def forward_gpu(self, inputs):
                 self.bias_desc = cudnn.create_tensor_descriptor(
                     b[None, :, None, None])
 
-            one = numpy.array(1, dtype=x.dtype).ctypes
-            zero = numpy.array(0, dtype=x.dtype).ctypes
+            oz_dtype = 'd' if x.dtype == 'd' else 'f'
+            one = numpy.array(1, dtype=oz_dtype).ctypes
+            zero = numpy.array(0, dtype=oz_dtype).ctypes
 
             if _cudnn_version >= 4000:
                 workspace_size = cuda.get_max_workspace_size()
@@ -146,10 +152,10 @@ def forward_gpu(self, inputs):
             W_mat = W.reshape(in_c, c * kh * kw)
             x_mats = x.reshape(n, in_c, in_h * in_w)
             gcol = cuda.cupy.empty(
-                (n, c, kh, kw, in_h, in_w), dtype=numpy.float32)
+                (n, c, kh, kw, in_h, in_w), dtype=x.dtype)
             gcol_mats = gcol.reshape(n, c * kh * kw, in_h * in_w)
             for i in moves.range(n):
-                cuda.cupy.dot(W_mat.T, x_mats[i], gcol_mats[i])
+                gcol_mats[i] = cuda.cupy.dot(W_mat.T, x_mats[i])
             y = conv.col2im_gpu(
                 gcol, self.sy, self.sx, self.ph, self.pw, self.outh, self.outw)
             if b is not None:
@@ -163,8 +169,8 @@ def backward_cpu(self, inputs, grad_outputs):
         kh, kw = W.shape[2:]
         col = conv.im2col_cpu(
             gy, kh, kw, self.sy, self.sx, self.ph, self.pw)
-        gW = numpy.tensordot(x, col, ([0, 2, 3], [0, 4, 5]))
-        gx = numpy.tensordot(col, W, ([1, 2, 3], [1, 2, 3]))
+        gW = numpy.tensordot(x, col, ([0, 2, 3], [0, 4, 5])).astype(W.dtype)
+        gx = numpy.tensordot(col, W, ([1, 2, 3], [1, 2, 3])).astype(x.dtype)
         gx = numpy.rollaxis(gx, 3, 1)
 
         if b is None:
@@ -180,9 +186,10 @@ def backward_gpu(self, inputs, grad_outputs):
         n, in_c, in_h, in_w = x.shape
         _, out_channels, kh, kw = W.shape
         c, h, w = gy.shape[1:]
-        gx = cuda.cupy.empty((n, in_c, in_h, in_w), dtype=numpy.float32)
+        gx = cuda.cupy.empty((n, in_c, in_h, in_w), dtype=x.dtype)
 
-        if cuda.cudnn_enabled and self.use_cudnn:
+        if (cuda.cudnn_enabled and self.use_cudnn and
+                _check_cudnn_acceptable_type(x.dtype, W.dtype)):
             x = cuda.cupy.ascontiguousarray(x)
             W = cuda.cupy.ascontiguousarray(W)
             gy = cuda.cupy.ascontiguousarray(gy)
@@ -199,8 +206,9 @@ def backward_gpu(self, inputs, grad_outputs):
                 workspace_size)
             workspace = cuda.cupy.empty((workspace_size,), dtype='b')
 
-            one = numpy.array(1, dtype=x.dtype).ctypes
-            zero = numpy.array(0, dtype=x.dtype).ctypes
+            oz_dtype = 'd' if x.dtype == 'd' else 'f'
+            one = numpy.array(1, dtype=oz_dtype).ctypes
+            zero = numpy.array(0, dtype=oz_dtype).ctypes
 
             libcudnn.convolutionForward(
                 handle, one.data, gy_desc.value, gy.data.ptr,
diff --git a/chainer/functions/connection/linear.py b/chainer/functions/connection/linear.py
--- a/chainer/functions/connection/linear.py
+++ b/chainer/functions/connection/linear.py
@@ -1,5 +1,3 @@
-import numpy
-
 from chainer import function
 from chainer.utils import type_check
 
@@ -18,8 +16,8 @@ def check_type_forward(self, in_types):
         x_type, w_type = in_types[:2]
 
         type_check.expect(
-            x_type.dtype == numpy.float32,
-            w_type.dtype == numpy.float32,
+            x_type.dtype.kind == 'f',
+            w_type.dtype.kind == 'f',
             x_type.ndim >= 2,
             w_type.ndim == 2,
             type_check.prod(x_type.shape[1:]) == w_type.shape[1],
@@ -27,7 +25,7 @@ def check_type_forward(self, in_types):
         if n_in.eval() == 3:
             b_type = in_types[2]
             type_check.expect(
-                b_type.dtype == numpy.float32,
+                b_type.dtype == x_type.dtype,
                 b_type.ndim == 1,
                 b_type.shape[0] == w_type.shape[0],
             )
@@ -35,7 +33,7 @@ def check_type_forward(self, in_types):
     def forward(self, inputs):
         x = _as_mat(inputs[0])
         W = inputs[1]
-        y = x.dot(W.T)
+        y = x.dot(W.T).astype(x.dtype)
         if len(inputs) == 3:
             b = inputs[2]
             y += b
@@ -46,8 +44,8 @@ def backward(self, inputs, grad_outputs):
         W = inputs[1]
         gy = grad_outputs[0]
 
-        gx = gy.dot(W).reshape(inputs[0].shape)
-        gW = gy.T.dot(x)
+        gx = gy.dot(W).astype(x.dtype).reshape(inputs[0].shape)
+        gW = gy.T.dot(x).astype(W.dtype)
         if len(inputs) == 3:
             gb = gy.sum(0)
             return gx, gW, gb


Test patch:
diff --git a/tests/chainer_tests/functions_tests/connection_tests/test_convolution_2d.py b/tests/chainer_tests/functions_tests/connection_tests/test_convolution_2d.py
--- a/tests/chainer_tests/functions_tests/connection_tests/test_convolution_2d.py
+++ b/tests/chainer_tests/functions_tests/connection_tests/test_convolution_2d.py
@@ -16,6 +16,8 @@
 @testing.parameterize(*testing.product({
     'c_contiguous': [True, False],
     'cover_all': [True, False],
+    'x_dtype': [numpy.float16, numpy.float32, numpy.float64],
+    'W_dtype': [numpy.float16, numpy.float32, numpy.float64],
 }))
 class TestConvolution2DFunction(unittest.TestCase):
 
@@ -28,18 +30,28 @@ def setUp(self, use_cudnn=True):
         self.use_cudnn = use_cudnn
         self.W = numpy.random.normal(
             0, numpy.sqrt(1. / (kh * kw * in_channels)),
-            (out_channels, in_channels, kh, kw)).astype(numpy.float32)
+            (out_channels, in_channels, kh, kw)).astype(self.W_dtype)
         self.b = numpy.random.uniform(
-            -1, 1, out_channels).astype(numpy.float32)
+            -1, 1, out_channels).astype(self.x_dtype)
 
-        self.x = numpy.random.uniform(-1, 1,
-                                      (2, 3, 4, 3)).astype(numpy.float32)
+        self.x = numpy.random.uniform(
+            -1, 1, (2, 3, 4, 3)).astype(self.x_dtype)
         if self.cover_all:
             self.gy = numpy.random.uniform(-1, 1,
-                                           (2, 2, 3, 2)).astype(numpy.float32)
+                                           (2, 2, 3, 2)).astype(self.x_dtype)
         else:
-            self.gy = numpy.random.uniform(-1, 1,
-                                           (2, 2, 2, 2)).astype(numpy.float32)
+            self.gy = numpy.random.uniform(
+                -1, 1, (2, 2, 2, 2)).astype(self.x_dtype)
+        self.check_forward_options = {}
+        self.check_backward_options = {'eps': 1e-2}
+        if self.x_dtype == numpy.float16:
+            self.check_forward_options = {'atol': 1e-4, 'rtol': 1e-3}
+            self.check_backward_options = {
+                'eps': 2 ** -3, 'atol': 1e-2, 'rtol': 1e-1}
+        elif self.W_dtype == numpy.float16:
+            self.check_forward_options = {'atol': 1e-4, 'rtol': 1e-3}
+            self.check_backward_options = {
+                'eps': 2 ** -3, 'atol': 1e-3, 'rtol': 1e-2}
 
     @attr.cudnn
     def test_forward_consistency(self, nobias=False):
@@ -57,7 +69,8 @@ def test_forward_consistency(self, nobias=False):
             x_gpu, W_gpu, b_gpu, stride=self.stride, pad=self.pad,
             use_cudnn=self.use_cudnn, cover_all=self.cover_all)
 
-        gradient_check.assert_allclose(y_cpu.data, y_gpu.data.get())
+        gradient_check.assert_allclose(
+            y_cpu.data, y_gpu.data.get(), **self.check_forward_options)
 
     @attr.gpu
     def test_forward_consistency_im2col(self):
@@ -91,7 +104,7 @@ def check_backward(self, x_data, W_data, b_data, y_grad):
         gradient_check.check_backward(
             convolution_2d.Convolution2DFunction(
                 self.stride, self.pad, self.use_cudnn, self.cover_all),
-            args, y_grad, eps=1e-2)
+            args, y_grad, **self.check_backward_options)
 
     @condition.retry(3)
     def test_backward_cpu(self):
@@ -130,6 +143,7 @@ def test_backward_gpu_im2col_nobias(self):
 
 @testing.parameterize(*testing.product({
     'use_cudnn': [True, False],
+    'dtype': [numpy.float16, numpy.float32, numpy.float64],
 }))
 @attr.cudnn
 class TestConvolution2DCudnnCall(unittest.TestCase):
@@ -141,12 +155,15 @@ def setUp(self):
         self.stride = 2
         self.pad = 1
         self.x = cuda.cupy.random.uniform(
-            -1, 1, (2, 3, 4, 3)).astype(numpy.float32)
+            -1, 1, (2, 3, 4, 3)).astype(self.dtype)
         self.W = cuda.cupy.random.normal(
             0, numpy.sqrt(1. / (kh * kw * in_channels)),
-            (out_channels, in_channels, kh, kw)).astype(numpy.float32)
+            (out_channels, in_channels, kh, kw)).astype(self.dtype)
         self.gy = cuda.cupy.random.uniform(
-            -1, 1, (2, 2, 2, 2)).astype(numpy.float32)
+            -1, 1, (2, 2, 2, 2)).astype(self.dtype)
+        self.expect = self.use_cudnn and (
+            cuda.cudnn.cudnn.getVersion() >= 3000 or
+            self.dtype != numpy.float16)
 
     def forward(self):
         x = chainer.Variable(self.x)
@@ -158,16 +175,18 @@ def forward(self):
     def test_call_cudnn_forward(self):
         with mock.patch('cupy.cudnn.cudnn.convolutionForward') as func:
             self.forward()
-            self.assertEqual(func.called, self.use_cudnn)
+            self.assertEqual(func.called, self.expect)
 
     def test_call_cudnn_backrward(self):
         y = self.forward()
         y.grad = self.gy
-        v2 = 'cupy.cudnn.cudnn.convolutionBackwardData_v2'
-        v3 = 'cupy.cudnn.cudnn.convolutionBackwardData_v3'
-        with mock.patch(v2) as func_v2, mock.patch(v3) as func_v3:
+        if cuda.cudnn.cudnn.getVersion() >= 4000:
+            name = 'cupy.cudnn.cudnn.convolutionBackwardData_v3'
+        else:
+            name = 'cupy.cudnn.cudnn.convolutionBackwardData_v2'
+        with mock.patch(name) as func:
             y.backward()
-            self.assertEqual(func_v2.called or func_v3.called, self.use_cudnn)
+            self.assertEqual(func.called, self.expect)
 
 
 testing.run_module(__name__, __file__)
diff --git a/tests/chainer_tests/functions_tests/connection_tests/test_deconvolution_2d.py b/tests/chainer_tests/functions_tests/connection_tests/test_deconvolution_2d.py
--- a/tests/chainer_tests/functions_tests/connection_tests/test_deconvolution_2d.py
+++ b/tests/chainer_tests/functions_tests/connection_tests/test_deconvolution_2d.py
@@ -21,20 +21,20 @@ def _pair(x):
     return x, x
 
 
-@parameterize(
-    *testing.product({
-        'in_channels': [3],
-        'out_channels': [2],
-        'wscale': [1],
-        'ksize': [3],
-        'stride': [1, 2],
-        'pad': [1],
-        'nobias': [True, False],
-        'use_cudnn': [True, False],
-        'test_outsize': [True, False],
-        'c_contiguous': [True, False],
-    })
-)
+@parameterize(*testing.product({
+    'in_channels': [3],
+    'out_channels': [2],
+    'wscale': [1],
+    'ksize': [3],
+    'stride': [1, 2],
+    'pad': [1],
+    'nobias': [True, False],
+    'use_cudnn': [True, False],
+    'test_outsize': [True, False],
+    'c_contiguous': [True, False],
+    'x_dtype': [numpy.float16, numpy.float32, numpy.float64],
+    'W_dtype': [numpy.float16, numpy.float32, numpy.float64],
+}))
 class TestDeconvolution2DFunction(unittest.TestCase):
 
     def setUp(self, use_cudnn=True):
@@ -44,9 +44,9 @@ def setUp(self, use_cudnn=True):
         self.W = numpy.random.normal(
             0, self.wscale * numpy.sqrt(1. / (kh * kw * self.in_channels)),
             (self.in_channels, self.out_channels, kh, kw)
-        ).astype(numpy.float32)
+        ).astype(self.W_dtype)
         self.b = None if self.nobias else numpy.random.uniform(
-            -1, 1, self.out_channels).astype(numpy.float32)
+            -1, 1, self.out_channels).astype(self.x_dtype)
 
         N = 2
         inh, inw = 4, 3
@@ -54,9 +54,18 @@ def setUp(self, use_cudnn=True):
         outw = conv.get_deconv_outsize(inw, kw, sw, pw)
         self.outsize = (outh, outw) if self.test_outsize else None
         self.x = numpy.random.uniform(
-            -1, 1, (N, self.in_channels, inh, inw)).astype(numpy.float32)
+            -1, 1, (N, self.in_channels, inh, inw)).astype(self.x_dtype)
         self.gy = numpy.random.uniform(
-            -1, 1, (N, self.out_channels, outh, outw)).astype(numpy.float32)
+            -1, 1, (N, self.out_channels, outh, outw)).astype(self.x_dtype)
+        self.test_forward_options = {}
+        self.check_backward_options = {'eps': 1e-2}
+        if self.x_dtype == numpy.float16:
+            self.test_forward_options = {'atol': 5e-3, 'rtol': 5e-2}
+            self.check_backward_options = {
+                'eps': 2**-3, 'atol': 1e-2, 'rtol': 1e-1}
+        elif self.W_dtype == numpy.float16:
+            self.check_backward_options = {
+                'eps': 2**-3, 'atol': 1e-3, 'rtol': 1e-2}
 
     @attr.cudnn
     def test_forward_consistency(self):
@@ -75,10 +84,14 @@ def test_forward_consistency(self):
             x_gpu, W_gpu, b_gpu, stride=self.stride, pad=self.pad,
             outsize=self.outsize, use_cudnn=self.use_cudnn)
 
-        gradient_check.assert_allclose(y_cpu.data, y_gpu.data.get())
+        self.assertEqual(y_cpu.data.dtype, self.x_dtype)
+        self.assertEqual(y_gpu.data.dtype, self.x_dtype)
+        gradient_check.assert_allclose(y_cpu.data, y_gpu.data.get(),
+                                       **self.test_forward_options)
 
     @attr.gpu
     def test_forward_consistency_im2col(self):
+        self.use_cudnn = False
         self.test_forward_consistency()
 
     def check_backward(self, x_data, W_data, b_data, y_grad):
@@ -103,24 +116,24 @@ def check_backward(self, x_data, W_data, b_data, y_grad):
         gradient_check.check_backward(
             deconvolution_2d.Deconvolution2DFunction(
                 self.stride, self.pad, self.outsize, self.use_cudnn),
-            args, y_grad, eps=1e-2)
+            args, y_grad, **self.check_backward_options)
 
-    @condition.retry(3)
+    @condition.retry(10)
     def test_backward_cpu(self):
         self.check_backward(self.x, self.W, self.b, self.gy)
 
     @attr.cudnn
-    @condition.retry(3)
+    @condition.retry(10)
     def test_backward_gpu(self):
         b = None if self.b is None else cuda.to_gpu(self.b)
         self.check_backward(cuda.to_gpu(self.x), cuda.to_gpu(self.W),
                             b, cuda.to_gpu(self.gy))
 
 
-@testing.parameterize(
-    {'use_cudnn': True},
-    {'use_cudnn': False},
-)
+@testing.parameterize(*testing.product({
+    'use_cudnn': [True, False],
+    'dtype': [numpy.float16, numpy.float32, numpy.float64],
+}))
 @attr.cudnn
 class TestDeconvolution2DCudnnCall(unittest.TestCase):
 
@@ -133,15 +146,18 @@ def setUp(self):
         self.W = cuda.cupy.random.normal(
             0, numpy.sqrt(1. / (kh * kw * self.in_channels)),
             (self.in_channels, self.out_channels, kh, kw)
-        ).astype(numpy.float32)
+        ).astype(self.dtype)
         N = 2
         inh, inw = 4, 3
         outh = conv.get_deconv_outsize(inh, kh, sh, ph)
         outw = conv.get_deconv_outsize(inw, kw, sw, pw)
         self.x = cuda.cupy.random.uniform(
-            -1, 1, (N, self.in_channels, inh, inw)).astype(numpy.float32)
+            -1, 1, (N, self.in_channels, inh, inw)).astype(self.dtype)
         self.gy = cuda.cupy.random.uniform(
-            -1, 1, (N, self.out_channels, outh, outw)).astype(numpy.float32)
+            -1, 1, (N, self.out_channels, outh, outw)).astype(self.dtype)
+        self.expect = self.use_cudnn and (
+            cuda.cudnn.cudnn.getVersion() >= 3000 or
+            self.dtype != numpy.float16)
 
     def forward(self):
         x = chainer.Variable(self.x)
@@ -150,18 +166,20 @@ def forward(self):
             x, W, None, stride=1, pad=1, use_cudnn=self.use_cudnn)
 
     def test_call_cudnn_forward(self):
-        v2 = 'cupy.cudnn.cudnn.convolutionBackwardData_v2'
-        v3 = 'cupy.cudnn.cudnn.convolutionBackwardData_v3'
-        with mock.patch(v2) as func_v2, mock.patch(v3) as func_v3:
+        if cuda.cudnn.cudnn.getVersion() >= 4000:
+            name = 'cupy.cudnn.cudnn.convolutionBackwardData_v3'
+        else:
+            name = 'cupy.cudnn.cudnn.convolutionBackwardData_v2'
+        with mock.patch(name) as func:
             self.forward()
-            self.assertEqual(func_v2.called or func_v3.called, self.use_cudnn)
+            self.assertEqual(func.called, self.expect)
 
-    def test_call_cudnn_backrward(self):
+    def test_call_cudnn_backward(self):
         y = self.forward()
         y.grad = self.gy
         with mock.patch('cupy.cudnn.cudnn.convolutionForward') as func:
             y.backward()
-            self.assertEqual(func.called, self.use_cudnn)
+            self.assertEqual(func.called, self.expect)
 
 
 testing.run_module(__name__, __file__)
diff --git a/tests/chainer_tests/functions_tests/connection_tests/test_linear.py b/tests/chainer_tests/functions_tests/connection_tests/test_linear.py
--- a/tests/chainer_tests/functions_tests/connection_tests/test_linear.py
+++ b/tests/chainer_tests/functions_tests/connection_tests/test_linear.py
@@ -12,17 +12,28 @@
 from chainer.testing import condition
 
 
+@testing.parameterize(*testing.product({
+    'x_dtype': [numpy.float16, numpy.float32, numpy.float64],
+    'W_dtype': [numpy.float16, numpy.float32, numpy.float64],
+}))
 class TestNonparameterizedLinear(unittest.TestCase):
 
     def setUp(self):
         self.W = numpy.random.uniform(
-            -1, 1, (2, 3)).astype(numpy.float32)
+            -1, 1, (2, 3)).astype(self.W_dtype)
         self.b = numpy.random.uniform(
-            -1, 1, 2).astype(numpy.float32)
+            -1, 1, 2).astype(self.x_dtype)
 
-        self.x = numpy.random.uniform(-1, 1, (4, 3)).astype(numpy.float32)
-        self.gy = numpy.random.uniform(-1, 1, (4, 2)).astype(numpy.float32)
+        self.x = numpy.random.uniform(-1, 1, (4, 3)).astype(self.x_dtype)
+        self.gy = numpy.random.uniform(-1, 1, (4, 2)).astype(self.x_dtype)
         self.y = self.x.dot(self.W.T) + self.b
+        self.check_forward_options = {}
+        self.check_backward_options = {}
+        if self.x_dtype == numpy.float16:
+            self.check_forward_options = {'atol': 1e-4, 'rtol': 1e-3}
+            self.check_backward_options = {'atol': 5e-2, 'rtol': 1e-1}
+        elif self.W_dtype == numpy.float16:
+            self.check_backward_options = {'atol': 5e-3, 'rtol': 5e-2}
 
     def check_forward(self, x_data, W_data, b_data, y_expect):
         x = chainer.Variable(x_data)
@@ -32,7 +43,9 @@ def check_forward(self, x_data, W_data, b_data, y_expect):
         else:
             b = chainer.Variable(b_data)
             y = functions.linear(x, W, b)
-        gradient_check.assert_allclose(y_expect, y.data)
+        self.assertEqual(y.data.dtype, self.x_dtype)
+        gradient_check.assert_allclose(
+            y_expect, y.data, **self.check_forward_options)
 
     @condition.retry(3)
     def test_forward_cpu(self):
@@ -63,7 +76,8 @@ def check_backward(self, x_data, W_data, b_data, y_grad):
             args = args + (b_data,)
 
         gradient_check.check_backward(
-            linear.LinearFunction(), args, y_grad, eps=1e-2)
+            linear.LinearFunction(), args, y_grad,
+            eps=1e-2, **self.check_backward_options)
 
     @condition.retry(3)
     def test_backward_cpu(self):
diff --git a/tests/chainer_tests/test_function.py b/tests/chainer_tests/test_function.py
--- a/tests/chainer_tests/test_function.py
+++ b/tests/chainer_tests/test_function.py
@@ -283,8 +283,19 @@ def test_backward(self):
 
 class TestFunctionInvalidType(unittest.TestCase):
 
-    def test_forward_invalid(self):
-        f = F.Linear(5, 5)
+    def test_forward_invalid1(self):
+        class Function(chainer.Function):
+            def check_type_forward(self, in_types):
+                x_type, = in_types
+                type_check.expect(
+                    x_type.dtype == numpy.float32,
+                    x_type.ndim >= 2,
+                )
+
+            def forward(self, inputs):
+                return inputs
+
+        f = Function()
 
         # OK
         v = chainer.Variable(numpy.random.randn(1, 5).astype(numpy.float32))
@@ -294,7 +305,7 @@ def test_forward_invalid(self):
         # Incorrect dtype
         # in py3, numpy dtypes are represented as class
         msg = """\
-Invalid operation is performed in: LinearFunction \\(Forward\\)
+Invalid operation is performed in: Function \\(Forward\\)
 
 Expect: in_types\\[0\\]\\.dtype == <(type|class) 'numpy\\.float32'>
 Actual: float64 \\!= <(type|class) 'numpy\\.float32'>"""
@@ -306,7 +317,7 @@ def test_forward_invalid(self):
 
         # Incorrect dim
         msg = """\
-Invalid operation is performed in: LinearFunction \\(Forward\\)
+Invalid operation is performed in: Function \\(Forward\\)
 
 Expect: in_types\\[0\\]\\.ndim >= 2
 Actual: 1 < 2"""