Repository: pytorch
Issue: RuntimeError: cuDNN error: CUDNN_STATUS_EXECUTION_FAILED
Labels: module: dependency bug,module: cudnn,module: cuda,module: convolution,triaged

### 🐛 Describe the bug

**🐛 Bug**
- cuDNN error on Torch 1.6.0 with CUDA 10.1 when running on a 1080Ti with 512.15 Nvidia drivers.
- cuDNN error on Torch 1.11.0 with CUDA 11.3 when running on a 1080Ti with 512.15 Nvidia drivers.

**To Reproduce**
Save this script as **resnet_lstm.py**
```
import torch
from torch import nn
import torch.nn.functional as F
import torchvision

class ResNetEncoder(nn.Module):
    """
    input shape = (batch_size, timestep, h, w, 3)
    output shape = (batch_size, timestep, output_dim)
    """
    def __init__(self, resnet_deepth=34, pretrain=True, fc_hidden1=512, fc_hidden2=256, dropout_prob=0.3, output_dim=128):
        super().__init__()
        assert resnet_deepth in [18, 34, 50, 101, 152]

        self.fc_hidden1, self.fc_hidden2 = fc_hidden1, fc_hidden2
        self.dropout_prob = dropout_prob

        # resnet = torch.hub.load('pytorch/vision:v0.10.0', f'resnet{resnet_deepth}', pretrained=pretrain)
        if resnet_deepth == 18:
            resnet = torchvision.models.resnet18(pretrained=pretrain, progress=True)
            print("loaded resnet18")
        if resnet_deepth == 34:
            resnet = torchvision.models.resnet34(pretrained=pretrain, progress=True)
            print("loaded resnet34")
        if resnet_deepth == 50:
            resnet = torchvision.models.resnet50(pretrained=pretrain, progress=True)
            print("loaded resnet50")
        if resnet_deepth == 101:
            resnet = torchvision.models.resnet101(pretrained=pretrain, progress=True)
            print("loaded resnet101")
        if resnet_deepth == 152:
            resnet = torchvision.models.resnet151(pretrained=pretrain, progress=True)
            print("loaded resnet152")

        modules = nn.ModuleList(resnet.children())[:-1]      # delete the last fc layer.
        self.resnet = nn.Sequential(*modules)
        self.fc1 = nn.Linear(resnet.fc.in_features, fc_hidden1)
        self.bn1 = nn.BatchNorm1d(fc_hidden1, momentum=0.01)
        self.fc2 = nn.Linear(fc_hidden1, fc_hidden2)
        self.bn2 = nn.BatchNorm1d(fc_hidden2, momentum=0.01)
        self.fc3 = nn.Linear(fc_hidden2, output_dim)

    def forward(self, x_3d):
        cnn_embed_seq = []
        for t in range(x_3d.size(1)):
            x = self.resnet(x_3d[:, t, :, :, :])  # ResNet
            x = x.view(x.size(0), -1)             # flatten output of conv

            x = self.fc1(x)
            x = self.bn1(x)
            x = nn.ReLU()(x)
            x = self.fc2(x)
            x = self.bn2(x)
            x = nn.ReLU()(x)
            x = nn.Dropout(p=self.dropout_prob)(x)
            x = self.fc3(x)

            cnn_embed_seq.append(x)

        # swap time and sample dim such that (sample dim, time dim, CNN latent dim)
        # torch.stack(cnn_embed_seq, dim=0): shape=(time_step, batch, input_size)
        cnn_embed_seq = torch.stack(cnn_embed_seq, dim=0)
        cnn_embed_seq = cnn_embed_seq.transpose_(0, 1)       
        
        # cnn_embed_seq: shape=(batch, time_step, input_size)
        return cnn_embed_seq

class LSTMDecoder(nn.Module):
    """
    input shape = (batch_size, timestep, input_dim)
    output shape = (batch_size, timestep, num_classes)
    """
    def __init__(self, input_dim=128,
                 num_lstm_layers=2,
                 lstm_hidden=128,
                 fc_dim=56,
                 dropout_prob=0.3,
                 num_classes=5):
        super().__init__()

        self.input_dim = input_dim
        self.num_lstm_layers = num_lstm_layers         # RNN hidden layers
        self.lstm_hidden = lstm_hidden                 # RNN hidden nodes
        self.fc_dim = fc_dim
        self.dropout_prob = dropout_prob
        self.num_classes = num_classes

        self.LSTM = nn.LSTM(
            input_size=self.input_dim,
            hidden_size=lstm_hidden,
            num_layers=num_lstm_layers,
            batch_first=True,
            bidirectional=True
        )

        self.fc1 = nn.Linear(self.lstm_hidden*2, self.fc_dim)
        self.fc2 = nn.Linear(self.fc_dim, self.num_classes)

    def forward(self, x):
        output, (_, _) = self.LSTM(x)
        # FC layers
        output = self.fc1(output)   # choose RNN_out at the last time step
        output = nn.ReLU()(output)
        output = nn.Dropout(p=self.dropout_prob)(output)
        output = self.fc2(output)

        # output = torch.sigmoid(output)
        # output = torch.unsqueeze(output, -1)
        return output
```

Save this script as test.py
```
import torch
import torch.nn as nn
from resnet_lstm import ResNetEncoder, LSTMDecoder

device = "cuda" if torch.cuda.is_available() else "cpu"

encoder = ResNetEncoder(resnet_deepth=50,
                        pretrain=True,
                        fc_hidden1=512,
                        fc_hidden2=128,
                        dropout_prob=0.3,
                        output_dim=128)

decoder = LSTMDecoder(input_dim=128,
                    num_lstm_layers=2,
                    lstm_hidden=128,
                    fc_dim=56,
                    dropout_prob=0.3,
                    num_classes=self.num_cls)
                    
encoder = encoder.to(device)
decoder = decoder.to(device)

self.encoder.eval()
self.decoder.eval()

for _ in range(10):
    batched_frames = torch.randn(16, 21, 3, 336, 336).to(device)
    with torch.no_grad():
        X_embedding = encoder(batched_frames)
        y_hat = decoder(X_embedding)
        y_hat = y_hat.reshape(-1, 21, 4)
        y_hat = torch.nn.Softmax(dim=-1)(y_hat).detach().cpu()
```



### Versions

Collecting environment information...
PyTorch version: 1.11.0
Is debug build: False
CUDA used to build PyTorch: 11.3
ROCM used to build PyTorch: N/A

OS: Ubuntu 18.04.5 LTS (x86_64)
GCC version: (Ubuntu 7.5.0-3ubuntu1~18.04) 7.5.0
Clang version: Could not collect
CMake version: Could not collect
Libc version: glibc-2.17

Python version: 3.7.13 (default, Mar 29 2022, 02:18:16)  [GCC 7.5.0] (64-bit runtime)
Python platform: Linux-4.19.128-microsoft-standard-x86_64-with-debian-buster-sid
Is CUDA available: True
CUDA runtime version: 11.1.105
GPU models and configuration: GPU 0: NVIDIA GeForce GTX 1080 Ti
Nvidia driver version: 512.15
cuDNN version: Could not collect
HIP runtime version: N/A
MIOpen runtime version: N/A

Versions of relevant libraries:
[pip3] numpy==1.19.0
[pip3] torch==1.11.0
[pip3] torchaudio==0.11.0
[pip3] torchvision==0.12.0
[conda] blas                      1.0                         mkl
[conda] cudatoolkit               11.3.1               h2bc3f7f_2
[conda] ffmpeg                    4.3                  hf484d3e_0    pytorch
[conda] mkl                       2021.4.0           h06a4308_640
[conda] mkl-service               2.4.0            py37h7f8727e_0
[conda] mkl_fft                   1.3.1            py37hd3c417c_0
[conda] mkl_random                1.2.2            py37h51133e4_0
[conda] numpy                     1.19.0                   pypi_0    pypi
[conda] pytorch                   1.11.0          py3.7_cuda11.3_cudnn8.2.0_0    pytorch
[conda] pytorch-mutex             1.0                        cuda    pytorch
[conda] torchaudio                0.11.0               py37_cu113    pytorch
[conda] torchvision               0.12.0               py37_cu113    pytorch

cc @csarofeen @ptrblck @xwang233 @ngimel