Repository: pytorch
Issue: model use dilated conv backward in v1.1.0 is ~3x slower than in v0.4.1 on 1080Ti 
Labels: module: performance,module: cudnn,module: cuda,module: convolution,triaged

## 🐛 Bug

<!-- A clear and concise description of what the bug is. -->

we found backward time of model with dilated conv in v1.1.0 much slower than in v0.4.1 on 1080ti, but comparable on v100.

## To Reproduce

```
import torch
import torch.nn as nn
import time


class Bottleneck(nn.Module):
    expansion = 4

    def __init__(self, inplanes, planes, stride=1, dilation=1, downsample=None):
        super(Bottleneck, self).__init__()
        assert dilation == 1 or (dilation > 1 and stride == 1)

        self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False)
        self.bn1 = nn.BatchNorm2d(planes)

        self.conv2 = nn.Conv2d(
            planes, planes, kernel_size=3,
            stride=stride, padding=dilation, dilation=dilation, bias=False)
        self.bn2 = nn.BatchNorm2d(planes)

        self.conv3 = nn.Conv2d(planes, planes * self.expansion, kernel_size=1, bias=False)
        self.bn3 = nn.BatchNorm2d(planes * self.expansion)

        self.relu = nn.ReLU(inplace=False)
        self.relu_inplace = nn.ReLU(inplace=True)
        self.downsample = downsample

    def forward(self, x):
        residual = x

        out = self.conv1(x)
        out = self.bn1(out)
        out = self.relu_inplace(out)

        out = self.conv2(out)
        out = self.bn2(out)
        out = self.relu_inplace(out)

        out = self.conv3(out)
        out = self.bn3(out)

        if self.downsample is not None:
            residual = self.downsample(x)

        out += residual
        out = self.relu_inplace(out)
        return out


class ResNet(nn.Module):
    def __init__(self, block, layers, use_stem=False, use_dilation=True):
        super(ResNet, self).__init__()

        self.use_stem = use_stem
        out_planes = [64, 64, 128, 256, 512]
        if use_stem:
            self.inplanes = 128
            self.conv1 = nn.Conv2d(3, 64, kernel_size=3, stride=2, padding=1, bias=False)
            self.bn1 = nn.BatchNorm2d(64)
            self.relu1 = nn.ReLU(inplace=False)

            self.conv2 = nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1, bias=False)
            self.bn2 = nn.BatchNorm2d(64)
            self.relu2 = nn.ReLU(inplace=False)

            self.conv3 = nn.Conv2d(64, 128, kernel_size=3, stride=1, padding=1, bias=False)
            self.bn3 = nn.BatchNorm2d(128)
            self.relu3 = nn.ReLU(inplace=False)
        else:
            self.inplanes = out_planes[0]
            self.conv1 = nn.Conv2d(3, out_planes[0], kernel_size=7, stride=2, padding=3, bias=False)
            self.bn1 = nn.BatchNorm2d(out_planes[0])
            self.relu1 = nn.ReLU(inplace=False)

        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1, ceil_mode=True)  # change ceil_mode

        self.layer1 = self._make_layer(block, out_planes[1], layers[0])
        self.layer2 = self._make_layer(block, out_planes[2], layers[1], stride=2)

        if use_dilation:
            self.layer3 = self._make_layer(block, out_planes[3], layers[2], stride=1, dilation=2)
            self.layer4 = self._make_layer(block, out_planes[4], layers[3], stride=1, dilation=4)
        else:
            self.layer3 = self._make_layer(block, out_planes[3], layers[2], stride=2)
            self.layer4 = self._make_layer(block, out_planes[4], layers[3], stride=2)

    def _make_layer(self, block, planes, blocks, stride=1, dilation=1):
        downsample = None
        if stride != 1 or self.inplanes != planes * block.expansion:
            downsample = nn.Sequential(
                nn.Conv2d(self.inplanes, planes * block.expansion, kernel_size=1, stride=stride, bias=False),
                nn.BatchNorm2d(planes * block.expansion),
            )

        layers = [block(self.inplanes, planes, stride, dilation, downsample)]
        self.inplanes = planes * block.expansion
        for i in range(1, blocks):
            layers.append(block(self.inplanes, planes, stride=1, dilation=dilation))
        return nn.Sequential(*layers)

    def forward(self, x):
        if self.use_stem:
            x = self.relu1(self.bn1(self.conv1(x)))
            x = self.relu2(self.bn2(self.conv2(x)))
            x = self.relu3(self.bn3(self.conv3(x)))
        else:
            x = self.relu1(self.bn1(self.conv1(x)))

        c1 = self.maxpool(x)

        c2 = self.layer1(c1)

        c3 = self.layer2(c2)

        c4 = self.layer3(c3)

        c5 = self.layer4(c4)

        return c5


if __name__ == '__main__':

    net = ResNet(Bottleneck, [3, 4, 6, 3]).cuda()

    x = torch.ones(1, 3, 1024, 2048).cuda()
    t0 = time.time()

    out = net(x)
    loss = out.sum()

    torch.cuda.synchronize()
    print("forward: ", time.time() - t0)

    t0 = time.time()
    
    loss.backward()

    torch.cuda.synchronize()

    print("backward: ", time.time() - t0)

```

v1.1.0 (cudnn_version=7501):
```
forward:  0.35411787033081055
backward:  1.4410080909729004
```
v0.4.1 (cudnn_version=7102):
```
forward:  0.37447357177734375
backward:  0.5850026607513428
```

<!-- If you have a code sample, error messages, stack traces, please provide it here as well -->

## Expected behavior

the backward time should be similar.

<!-- A clear and concise description of what you expected to happen. -->

## Environment

cuda 9.0
python 3.6

## Additional context

<!-- Add any other context about the problem here. -->