Repository: pytorch
Issue: AMP much worse performance with groupped Conv2d than fp32 
Labels: module: dependency bug,module: performance,module: cuda,triaged,module: amp (automated mixed precision)

It seems automatic mix precision on conv2d with groups>1 is significantly slower than fp32
This result is more obvious on machines with Volta GPUs with Tensor cores.

```
import torch
import torch.nn as nn
from torch.cuda.amp import autocast
import time


class amp_net(nn.Module):
    def __init__(self, amp, group):
        super().__init__()
        self.amp = amp
        if group:
            self.net = nn.Sequential(nn.Conv2d(12, 64, kernel_size=3, stride=2, padding=1, groups=4),
                                     nn.ReLU(),
                                     nn.Conv2d(64, 128, kernel_size=3, stride=2, padding=1, groups=4),
                                     nn.ReLU(),
                                     nn.Conv2d(128, 256, kernel_size=3, stride=2, padding=1, groups=4),
                                     nn.ReLU(),
                                     nn.Conv2d(256, 512, kernel_size=3, stride=2, padding=1, groups=4),
                                     nn.ReLU(),
                                     nn.Conv2d(512, 1024, kernel_size=3, stride=2, padding=1, groups=4),
                                     nn.Flatten(),
                                     nn.Linear(9216, 5)
                                     ).cuda()
        else:
            self.net = nn.Sequential(nn.Conv2d(12, 64, kernel_size=3, stride=2, padding=1),
                                     nn.ReLU(),
                                     nn.Conv2d(64, 128, kernel_size=3, stride=2, padding=1),
                                     nn.ReLU(),
                                     nn.Conv2d(128, 256, kernel_size=3, stride=2, padding=1),
                                     nn.ReLU(),
                                     nn.Conv2d(256, 512, kernel_size=3, stride=2, padding=1),
                                     nn.ReLU(),
                                     nn.Conv2d(512, 1024, kernel_size=3, stride=2, padding=1),
                                     nn.Flatten(),
                                     nn.Linear(9216, 5)
                                     ).cuda()

    def forward(self, x):
        with autocast(self.amp):
            return self.net(x)


def test_with_arch(group):
    net_amp = amp_net(True, group)
    optimizer_amp = torch.optim.Adam(net_amp.parameters(), lr=1e-3)
    net = amp_net(False, group)
    optimizer = torch.optim.Adam(net.parameters(), lr=1e-3)
    amp_scaler = torch.cuda.amp.GradScaler()

    sample_input = torch.rand((64, 12, 84, 84)).cuda()
    sample_output = torch.randint(5, (64, )).cuda()
    criteria = nn.CrossEntropyLoss()

    start_time = time.time()
    for _ in range(50):
        with torch.cuda.amp.autocast(True):
            result = net_amp(sample_input)
            loss = criteria(result, sample_output)
        optimizer_amp.zero_grad()
        amp_scaler.scale(loss).backward()
        amp_scaler.unscale_(optimizer_amp)
        amp_scaler.step(optimizer_amp)
        amp_scaler.update()

    end_time = time.time()
    time_amp = end_time - start_time

    start_time = time.time()
    for _ in range(50):
        with torch.cuda.amp.autocast(False):
            result = net(sample_input)
            loss = criteria(result, sample_output)
        optimizer.zero_grad()
        amp_scaler.scale(loss).backward()
        amp_scaler.unscale_(optimizer)
        amp_scaler.step(optimizer)
        amp_scaler.update()
    end_time = time.time()
    time_fp32 = end_time - start_time

    prefix = 'Group Conv2d: ' if group else 'Conv2d: '
    print(prefix + 'amp time / fp32 time {}'.format(time_amp / time_fp32))


if __name__ == '__main__':
    test_with_arch(False)
    test_with_arch(True)

```


Result:
```
Conv2d: amp time / fp32 time 0.3734461807645061
Group Conv2d: amp time / fp32 time 3.4929036465152117
```

cc @mcarilli @ngimel @VitalyFedyunin