<PyTorch tutorial> cuDNN error: CUDNN_STATUS_EXECUTION_FAILED

Lee11: <p>Hello, I’m a beginner of PyTorch.<br>
Now, I’m studying with the tutorial of PyTorch (Transfer learning tutorial).<br>
When I ran following code, I got an error, ‘RuntimeError: cuDNN error: CUDNN_STATUS_EXECUTION_FAILED’.<br>
Please let me know what is wrong in the code or in my running environment.<br>
My setting is<br>
GPU: Geforce RTX 2070<br>
CUDA ver: 9.0<br>
cuDNN ver: 7.5 (7005)</p>
<p>code:</p>
<p>from <strong>future</strong> import print_function, division</p>
<p>import torch<br>
import torch.nn as nn<br>
import torch.optim as optim<br>
from torch.optim import lr_scheduler<br>
from torch.autograd import Variable<br>
import numpy as np<br>
import torchvision<br>
from torchvision import datasets, models, transforms<br>
import matplotlib.pyplot as plt<br>
import time<br>
import os<br>
import copy</p>
<p>plt.ion()   # interactive mode</p>
<p>data_transforms = {<br>
‘train’: transforms.Compose([<br>
transforms.RandomResizedCrop(224),<br>
transforms.RandomHorizontalFlip(),<br>
transforms.ToTensor(),<br>
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])<br>
]),<br>
‘val’: transforms.Compose([<br>
transforms.Resize(256),<br>
transforms.CenterCrop(224),<br>
transforms.ToTensor(),<br>
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])<br>
]),<br>
}</p>
<p>data_dir = ‘hymenoptera_data’<br>
image_datasets = {x: datasets.ImageFolder(os.path.join(data_dir, x),<br>
data_transforms[x])<br>
for x in [‘train’, ‘val’]}<br>
dataloaders = {x: torch.utils.data.DataLoader(image_datasets[x], batch_size=4,<br>
shuffle=True, num_workers=4)<br>
for x in [‘train’, ‘val’]}<br>
dataset_sizes = {x: len(image_datasets[x]) for x in [‘train’, ‘val’]}<br>
class_names = image_datasets[‘train’].classes</p>
<p>use_gpu = torch.cuda.is_available()</p>
<p>def imshow(inp, title=None):<br>
“”“Imshow for Tensor.”""<br>
inp = inp.numpy().transpose((1, 2, 0))<br>
mean = np.array([0.485, 0.456, 0.406])<br>
std = np.array([0.229, 0.224, 0.225])<br>
inp = std * inp + mean<br>
inp = np.clip(inp, 0, 1)<br>
plt.imshow(inp)<br>
if title is not None:<br>
plt.title(title)<br>
plt.pause(0.1)  # pause a bit so that plots are updated</p>
<p>def train_model(model, criterion, optimizer, scheduler, num_epochs = 25):<br>
since = time.time()</p>
<pre><code>best_model_wts = copy.deepcopy(model.state_dict())
best_acc = 0.0

for epoch in range(num_epochs):
    print('Epoch {}/{}'.format(epoch, num_epochs - 1))
    print('-' * 10)

    # Each epoch has a training and validation phase
    for phase in ['train', 'val']:
        if phase == 'train':
            scheduler.step()
            model.train(True)  # Set model to training mode
        else:
            model.train(False)  # Set model to evaluate mode

        running_loss = 0.0
        running_corrects = 0

        # Iterate over data.
        for data in dataloaders[phase]:
            # get the inputs
            inputs, labels = data

            # wrap them in Variable
            if use_gpu:
                inputs = Variable(inputs.cuda())
                labels = Variable(labels.cuda())
            else:
                inputs, labels = Variable(inputs), Variable(labels)

            # zero the parameter gradients
            optimizer.zero_grad()

            # forward
            outputs = model(inputs)
            _, preds = torch.max(outputs.data, 1)
            loss = criterion(outputs, labels)

            # backward + optimize only if in training phase
            if phase == 'train':
                loss.backward()
                optimizer.step()

            # statistics
            running_loss += loss.data[0] * inputs.size(0)
            running_corrects += torch.sum(preds == labels.data)

        epoch_loss = running_loss / dataset_sizes[phase]
        epoch_acc = running_corrects / dataset_sizes[phase]

        print('{} Loss: {:.4f} Acc: {:.4f}'.format(
            phase, epoch_loss, epoch_acc))

        # deep copy the model
        if phase == 'val' and epoch_acc &gt; best_acc:
            best_acc = epoch_acc
            best_model_wts = copy.deepcopy(model.state_dict())

    print()

time_elapsed = time.time() - since
print('Training complete in {:.0f}m {:.0f}s'.format(
    time_elapsed // 60, time_elapsed % 60))
print('Best val Acc: {:4f}'.format(best_acc))

# load best model weights
model.load_state_dict(best_model_wts)
return model
</code></pre>
<p>def visualize_model(model, num_images=6):<br>
was_training = model.training<br>
model.eval()<br>
images_so_far = 0<br>
fig = plt.figure()</p>
<pre><code>for i, data in enumerate(dataloaders['val']):
    inputs, labels = data
    if use_gpu:
        inputs, labels = Variable(inputs.cuda()), Variable(labels.cuda())
    else:
        inputs, labels = Variable(inputs), Variable(labels)

    outputs = model(inputs)
    _, preds = torch.max(outputs.data, 1)

    for j in range(inputs.size()[0]):
        images_so_far += 1
        ax = plt.subplot(num_images//2, 2, images_so_far)
        ax.axis('off')
        ax.set_title('predicted: {}'.format(class_names[preds[j]]))
        imshow(inputs.cpu().data[j])

        if images_so_far == num_images:
            model.train(mode=was_training)
            return
model.train(mode=was_training)
</code></pre>
<p>def main():<br>
# Get a batch of training data<br>
inputs, classes = next(iter(dataloaders[‘train’]))</p>
<pre><code># Make a grid from batch
out = torchvision.utils.make_grid(inputs)

# imshow(out, title=[class_names[x] for x in classes])

model_ft = models.resnet18(pretrained=True)
num_ftrs = model_ft.fc.in_features
model_ft.fc = nn.Linear(num_ftrs, 2)

if use_gpu:
    model_ft = model_ft.cuda()

criterion = nn.CrossEntropyLoss()

# Observe that all parameters are being optimized
optimizer_ft = optim.SGD(model_ft.parameters(), lr=0.001, momentum=0.9)

# Decay LR by a factor of 0.1 every 7 epochs
exp_lr_scheduler = lr_scheduler.StepLR(optimizer_ft, step_size=7, gamma=0.1)

model_ft = train_model(model_ft, criterion, optimizer_ft, exp_lr_scheduler,
                       num_epochs=25)
</code></pre>
<p>if <strong>name</strong> == ‘<strong>main</strong>’:<br>
main()</p>
<p>error:</p>
<p>Traceback (most recent call last):<br>
File “D:/Research/Deep_learning/Study/Ex_Pytorch4/Transfer_Learning_1.py”, line 194, in <br>
main()<br>
File “D:/Research/Deep_learning/Study/Ex_Pytorch4/Transfer_Learning_1.py”, line 190, in main<br>
num_epochs=25)<br>
File “D:/Research/Deep_learning/Study/Ex_Pytorch4/Transfer_Learning_1.py”, line 100, in train_model<br>
outputs = model(inputs)<br>
File “C:\Users\Wonjune\Anaconda3\envs\pytorch\lib\site-packages\torch\nn\modules\module.py”, line 489, in <strong>call</strong><br>
result = self.forward(*input, **kwargs)<br>
File “C:\Users\Wonjune\Anaconda3\envs\pytorch\lib\site-packages\torchvision\models\resnet.py”, line 150, in forward<br>
x = self.conv1(x)<br>
File “C:\Users\Wonjune\Anaconda3\envs\pytorch\lib\site-packages\torch\nn\modules\module.py”, line 489, in <strong>call</strong><br>
result = self.forward(*input, **kwargs)<br>
File “C:\Users\Wonjune\Anaconda3\envs\pytorch\lib\site-packages\torch\nn\modules\conv.py”, line 320, in forward<br>
self.padding, self.dilation, self.groups)<br>
RuntimeError: cuDNN error: CUDNN_STATUS_EXECUTION_FAILED</p>

ptrblck: <p>Is your code running fine of the CPU?<br>
If so, could you disable cuDNN and try to run your code again using <code>torch.backends.cudnn.enabled = False</code>?</p>

a4b0841fd08898cbdaa2: <p>I have just solved this problem before, but I don’t know if it is suitable for you. Please check your following aspects:<br>
1 The type of data you entered, LongTensor or something else<br>
2 Is the shape of your neural network the same as the shape of your data?</p>

Lee11: <p>Thank you all, I solved the problem.<br>
All of your advice were helpful~!</p>