Repository: Theano/Theano

Issue:
RuntimeError: error getting worksize: CUDNN_STATUS_BAD_PARAM because of dtype
The code below will throw a CUDNN_STATUS_BAD_PARAM. The error can be removed by casting
`C` into a `float32` to not cause a division by an `int64` which will in return give a `float64` result. I did not do an extensive job at isolating exactly when the error is triggered. I tried to remove everything and only have a `conv3d` with a similar `float64` issue but the optimization passed and there were no error. So something seems to be failing somewhere when the graph get bigger.

It would nice if such a thing could be caught in the optimization phase or at least in a warning :)

Thanks!
using Theano at commit: e0496181f9b6a7ff58579bd4841b34886ca4f7cc

```
import numpy as np
import theano
import theano.tensor as T

ftensor5 = T.TensorType('float32', (False,)*5)
# assumes this comes into tbc01
in_x = ftensor5('x')
x = in_x

C = x.shape[-1] * x.shape[-2]
# Conv3D op wants bct01
# tbc01 -> bct01
x = x.dimshuffle(1,2,0,3,4)
C *= x.shape[1]

# ---> uncomment this cast to remove the error <---
#C = T.cast(C, 'float32')

g_W = np.random.random((128,128,1,1,1)).astype(np.float32)
theta_W = np.random.random((128,128,1,1,1)).astype(np.float32)
phi_W = np.random.random((128,128,1,1,1)).astype(np.float32)
z_W = np.random.random((128,128,1,1,1)).astype(np.float32)

g_W = theano.shared(g_W, name='g_W')
theta_W = theano.shared(theta_W, name='theta_W')
phi_W = theano.shared(phi_W, name='phi_W')
z_W = theano.shared(z_W, name='z_W')

def conv3d(y, W):
    border_mode = (0,0,0)
    subsample = (1,1,1)
    rval = T.nnet.conv3d(y, W, border_mode=border_mode, subsample=subsample, filter_flip=False)
    return rval

g_out = conv3d(x, g_W)
theta_out = conv3d(x, theta_W)
phi_out = conv3d(x, phi_W)
outs = [g_out, theta_out, phi_out]

b_shp = x.shape[0]
t_shp = (x.shape[2],)
c_shp = g_out.shape[1] # g, phi, and theta outs should all have the same channel shape
h_shp = x.shape[-2]
w_shp = x.shape[-1]

# bct01 -> bt01c
pattern = (0,2,3,4,1)
outs = map(lambda x: x.dimshuffle(*pattern), outs)

# collapse 01 together
outs = map(lambda x: x.reshape((b_shp,) + t_shp + (h_shp * w_shp, c_shp)), outs)

# collapse with t
outs = map(lambda x: x.reshape((b_shp, t_shp[0] * h_shp * w_shp, c_shp)), outs)

# now everything is in 3D: (b, t01, c)
assert all(y.ndim == 3 for y in outs)
thetaXphi = T.batched_dot(outs[1], outs[2].dimshuffle(0,2,1)) / C
out = T.batched_dot(thetaXphi, outs[0])

# unpack
out = out.dimshuffle(0,2,1)
out = out.reshape((out.shape[0], out.shape[1], t_shp[0], h_shp * w_shp))
out = out.reshape((out.shape[0], out.shape[1], out.shape[2], h_shp, w_shp))

# should be back to bct01
# residual connection
out = conv3d(out, z_W) + x

# bct01 -> tbc01
out = out.dimshuffle(2,0,1,3,4)

f = theano.function([in_x], [out])
npx = np.random.random((5,32,128,15,15)).astype(np.float32)
print f(npx)[0].shape
```


Discussion:

Resolving patch:
diff --git a/theano/gpuarray/dnn.py b/theano/gpuarray/dnn.py
--- a/theano/gpuarray/dnn.py
+++ b/theano/gpuarray/dnn.py
@@ -947,10 +947,14 @@ def _dnn_conv(img, kerns, alpha=1, beta=0, out=None, border_mode='valid', subsam
               conv_mode='conv', algo=None, precision=None, num_groups=1):
     ctx_name = infer_context_name(img, kerns)
 
-    img = gpu_contiguous(as_gpuarray_variable(img, ctx_name))
-    kerns = gpu_contiguous(as_gpuarray_variable(kerns, ctx_name))
+    img = as_gpuarray_variable(img, ctx_name)
+    kerns = as_gpuarray_variable(kerns, ctx_name)
 
     precision = get_precision(precision, [img, kerns])
+
+    img = gpu_contiguous(img.astype(precision))
+    kerns = gpu_contiguous(kerns.astype(precision))
+
     desc = GpuDnnConvDesc(border_mode=border_mode, subsample=subsample, dilation=dilation,
                           conv_mode=conv_mode, precision=precision, num_groups=num_groups)(kerns.shape)
     desc_op = desc.owner.op
@@ -974,11 +978,15 @@ def _dnn_gradweight(img, topgrad, kerns_shp, alpha=1, beta=0, out=None, border_m
                     dilation=(1, 1), conv_mode='conv', algo=None, precision=None, num_groups=1):
     ctx_name = infer_context_name(img, topgrad)
 
-    img = gpu_contiguous(as_gpuarray_variable(img, ctx_name))
-    topgrad = gpu_contiguous(as_gpuarray_variable(topgrad, ctx_name))
+    img = as_gpuarray_variable(img, ctx_name)
+    topgrad = as_gpuarray_variable(topgrad, ctx_name)
     kerns_shp = theano.tensor.as_tensor_variable(kerns_shp)
 
     precision = get_precision(precision, [img, topgrad], for_grad=True)
+
+    img = gpu_contiguous(img.astype(precision))
+    topgrad = gpu_contiguous(topgrad.astype(precision))
+
     desc = GpuDnnConvDesc(border_mode=border_mode, subsample=subsample, dilation=dilation,
                           conv_mode=conv_mode, precision=precision, num_groups=num_groups)(kerns_shp)
     if beta == 0:
@@ -995,11 +1003,15 @@ def _dnn_gradinput(kerns, topgrad, img_shp, alpha=1, beta=0, out=None, border_mo
                    dilation=(1, 1), conv_mode='conv', algo=None, precision=None, num_groups=1):
     ctx_name = infer_context_name(kerns, topgrad)
 
-    kerns = gpu_contiguous(as_gpuarray_variable(kerns, ctx_name))
-    topgrad = gpu_contiguous(as_gpuarray_variable(topgrad, ctx_name))
+    kerns = as_gpuarray_variable(kerns, ctx_name)
+    topgrad = as_gpuarray_variable(topgrad, ctx_name)
     img_shp = theano.tensor.as_tensor_variable(img_shp)
 
     precision = get_precision(precision, [kerns, topgrad], for_grad=True)
+
+    kerns = gpu_contiguous(kerns.astype(precision))
+    topgrad = gpu_contiguous(topgrad.astype(precision))
+
     desc = GpuDnnConvDesc(border_mode=border_mode, subsample=subsample, dilation=dilation,
                           conv_mode=conv_mode, precision=precision, num_groups=num_groups)(kerns.shape)
     if beta == 0:


Test patch:
diff --git a/theano/gpuarray/tests/test_dnn.py b/theano/gpuarray/tests/test_dnn.py
--- a/theano/gpuarray/tests/test_dnn.py
+++ b/theano/gpuarray/tests/test_dnn.py
@@ -220,6 +220,48 @@ def test_dnn_conv_invalid_precision():
     yield (run_dnn_conv_invalid_precision, 3)
 
 
+def test_dnn_conv_mixed_dtype():
+    mf = T.ftensor4()
+    md = T.dtensor4()
+
+    def assert_types(conv):
+        dt = conv.owner.inputs[0].dtype
+        assert conv.owner.inputs[1].dtype == dt
+        assert conv.owner.inputs[2].dtype == dt
+
+    assert_types(dnn.dnn_conv(md, mf, precision='as_input'))
+    assert_types(dnn.dnn_conv(mf, md, precision='as_input'))
+    assert_types(dnn.dnn_gradweight(mf, md, kerns_shp=mf.shape,
+                                    precision='as_input'))
+    assert_types(dnn.dnn_gradweight(md, mf, kerns_shp=mf.shape,
+                                    precision='as_input'))
+    assert_types(dnn.dnn_gradinput(mf, md, img_shp=mf.shape,
+                                   precision='as_input'))
+    assert_types(dnn.dnn_gradinput(md, mf, img_shp=mf.shape,
+                                   precision='as_input'))
+
+
+def test_dnn_conv3d_mixed_dtype():
+    mf = T.ftensor5()
+    md = T.dtensor5()
+
+    def assert_types(conv):
+        dt = conv.owner.inputs[0].dtype
+        assert conv.owner.inputs[1].dtype == dt
+        assert conv.owner.inputs[2].dtype == dt
+
+    assert_types(dnn.dnn_conv3d(md, mf, precision='as_input'))
+    assert_types(dnn.dnn_conv3d(mf, md, precision='as_input'))
+    assert_types(dnn.dnn_gradweight3d(mf, md, kerns_shp=mf.shape,
+                                      precision='as_input'))
+    assert_types(dnn.dnn_gradweight3d(md, mf, kerns_shp=mf.shape,
+                                      precision='as_input'))
+    assert_types(dnn.dnn_gradinput3d(mf, md, img_shp=mf.shape,
+                                     precision='as_input'))
+    assert_types(dnn.dnn_gradinput3d(md, mf, img_shp=mf.shape,
+                                     precision='as_input'))
+
+
 def test_pooling():
     if not dnn.dnn_available(test_ctx_name):
         raise SkipTest(dnn.dnn_available.msg)