Repository: pytorch
Issue: RuntimeError: CUDNN_STATUS_BAD_PARAM when testing environment
Labels: oncall: distributed

full repo:
`
Traceback (most recent call last):
  File "C:/Users/ps/PycharmProjects/pythonProject2/main.py", line 98, in <module>
    outputs = model(images)
  File "C:\Program Files\Python37\lib\site-packages\torch\nn\modules\module.py", line 727, in _call_impl
    result = self.forward(*input, **kwargs)
  File "C:\Program Files\Python37\lib\site-packages\torch\nn\parallel\data_parallel.py", line 161, in forward
    outputs = self.parallel_apply(replicas, inputs, kwargs)
  File "C:\Program Files\Python37\lib\site-packages\torch\nn\parallel\data_parallel.py", line 171, in parallel_apply
    return parallel_apply(replicas, inputs, kwargs, self.device_ids[:len(replicas)])
  File "C:\Program Files\Python37\lib\site-packages\torch\nn\parallel\parallel_apply.py", line 86, in parallel_apply
    output.reraise()
  File "C:\Program Files\Python37\lib\site-packages\torch\_utils.py", line 428, in reraise
    raise self.exc_type(msg)
RuntimeError: Caught RuntimeError in replica 1 on device 1.
Original Traceback (most recent call last):
  File "C:\Program Files\Python37\lib\site-packages\torch\nn\parallel\parallel_apply.py", line 61, in _worker
    output = module(*input, **kwargs)
  File "C:\Program Files\Python37\lib\site-packages\torch\nn\modules\module.py", line 727, in _call_impl
    result = self.forward(*input, **kwargs)
  File "C:/Users/ps/PycharmProjects/pythonProject2/main.py", line 60, in forward
    x = F.max_pool2d(F.relu(self.conv2(x)), (2, 2))
  File "C:\Program Files\Python37\lib\site-packages\torch\nn\modules\module.py", line 727, in _call_impl
    result = self.forward(*input, **kwargs)
  File "C:\Program Files\Python37\lib\site-packages\torch\nn\modules\conv.py", line 423, in forward
    return self._conv_forward(input, self.weight)
  File "C:\Program Files\Python37\lib\site-packages\torch\nn\modules\conv.py", line 420, in _conv_forward
    self.padding, self.dilation, self.groups)
RuntimeError: cuDNN error: CUDNN_STATUS_BAD_PARAM
You can try to repro this exception using the following code snippet. If that doesn't trigger the error, please include your original repro script when reporting this issue.

import torch
torch.backends.cuda.matmul.allow_tf32 = True
torch.backends.cudnn.benchmark = False
torch.backends.cudnn.deterministic = False
torch.backends.cudnn.allow_tf32 = True
data = torch.randn([32, 6, 14, 14], dtype=torch.float, device='cuda', requires_grad=True)
net = torch.nn.Conv2d(6, 16, kernel_size=[5, 5], padding=[0, 0], stride=[1, 1], dilation=[1, 1], groups=1)
net = net.cuda().float()
out = net(data)
out.backward(torch.randn_like(out))
torch.cuda.synchronize()

ConvolutionParams 
    data_type = CUDNN_DATA_FLOAT
    padding = [0, 0, 0]
    stride = [1, 1, 0]
    dilation = [1, 1, 0]
    groups = 1
    deterministic = false
    allow_tf32 = true
input: TensorDescriptor 00000000CECF21B0
    type = CUDNN_DATA_FLOAT
    nbDims = 4
    dimA = 32, 6, 14, 14, 
    strideA = 1176, 196, 14, 1, 
output: TensorDescriptor 00000000CECF2220
    type = CUDNN_DATA_FLOAT
    nbDims = 4
    dimA = 32, 16, 10, 10, 
    strideA = 1600, 100, 10, 1, 
weight: FilterDescriptor 00000000CC610130
    type = CUDNN_DATA_FLOAT
    tensor_format = CUDNN_TENSOR_NCHW
    nbDims = 4
    dimA = 16, 6, 5, 5, 
Pointer addresses: 
    input: 0000000B29B7AE00
    output: 0000000B6FE00000
    weight: 0000000B29A18A70
Forward algorithm: 1
`

full code:
`import torch
import torch.nn as nn
import torchvision
import time
import torch.nn.functional as F
import torchvision.transforms as transforms
if __name__ == '__main__':

    # Device configuration
    device = torch.device('cuda')

    # Hyper-parameters
    input_size = 784
    hidden_size = 3000
    num_classes = 10
    num_epochs = 2
    batch_size = 64
    learning_rate = 0.001

    # MNIST dataset
    train_dataset = torchvision.datasets.MNIST(root='../../data',
                                               train=True,
                                               transform=transforms.ToTensor(),
                                               download=True)

    test_dataset = torchvision.datasets.MNIST(root='../../data',
                                              train=False,
                                              transform=transforms.ToTensor())

    # Data loader
    train_loader = torch.utils.data.DataLoader(dataset=train_dataset,
                                               batch_size=batch_size,
                                               shuffle=True, num_workers=2)

    test_loader = torch.utils.data.DataLoader(dataset=test_dataset,
                                              batch_size=batch_size,
                                              shuffle=False, num_workers=2)


    # Fully connected neural network with one hidden layer
    class LeNet5(nn.Module):
        def __init__(self):
            super().__init__()
            self.conv1 = nn.Conv2d(1, 6, 5, padding=2)
            self.conv2 = nn.Conv2d(6, 16, 5)

            self.fc1 = nn.Linear(16 * 5 * 5, 120)
            self.fc2 = nn.Linear(120, 84)
            self.fc3 = nn.Linear(84, 10)

        def forward(self, x):
            x = F.max_pool2d(F.relu(self.conv1(x)), (2, 2))
            x = F.max_pool2d(F.relu(self.conv2(x)), (2, 2))
            x = x.view(-1, self.num_flat_features(x))
            x = F.relu(self.fc1(x))
            x = F.relu(self.fc2(x))
            x = self.fc3(x)
            return x

        def num_flat_features(self, x):
            size = x.size()[1:]
            num_features = 1
            for s in size:
                num_features *= s
            return num_features


    model = LeNet5()

    if torch.cuda.device_count() > 1 and True:
        model = nn.DataParallel(model, device_ids=[0, 1])
    model.to(device)
    # model = torch.nn.DataParallel(model)

    # Loss and optimizer
    criterion = nn.CrossEntropyLoss()
    optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)
    s = time.clock()
    # Train the model
    total_step = len(train_loader)
    for epoch in range(num_epochs):
        for i, (images, labels) in enumerate(train_loader):
            # Move tensors to the configured device
            images = images.reshape(-1, 1, 28, 28).float().to(device)
            labels = labels.float().to(device)

            # Forward pass
            outputs = model(images)
            loss = criterion(outputs, labels)

            # Backward and optimize
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()

            if (i + 1) % 100 == 0:
                print('Epoch [{}/{}], Step [{}/{}], Loss: {:.4f}'
                      .format(epoch + 1, num_epochs, i + 1, total_step, loss.item()))

    # Test the model
    # In test phase, we don't need to compute gradients (for memory efficiency)
    with torch.no_grad():
        correct = 0
        total = 0
        for images, labels in test_loader:
            images = images.reshape(-1, 1, 28, 28).to(device)
            labels = labels.to(device)
            outputs = model(images)
            _, predicted = torch.max(outputs.data, 1)
            total += labels.size(0)
            correct += (predicted == labels).sum().item()

        print('Accuracy of the network on the 10000 test images: {} %'.format(100 * correct / total))
`

environment: python3.7.7, 2080ti +2070s, cuda:10.2, pycharm: 2020.1
It ran correctly when I use cpu or single gpu, but as the number of gpus gets to 2, the error occured

cc @pietern @mrshenli @pritamdamania87 @zhaojuanmao @satgera @rohan-varma @gqchen @aazzolini @osalpekar @jiayisuse @agolynski @SciPioneer @H-Huang @mrzzd