The conv size mismatch!

lililila: <p>Hi,I encountered the following problem when trying to replace the dilated cconvolution in deeplabv3+ with  asymmetric convolution.Here is my code and the error reported：</p>
<p>import torch<br>
import torch.nn as nn<br>
import torch.nn.functional as F<br>
from nets.xception import xception<br>
from nets.mobilenetv2 import mobilenetv2</p>
<p>class MobileNetV2(nn.Module):<br>
def <strong>init</strong>(self, downsample_factor=8, pretrained=True):<br>
super(MobileNetV2, self).<strong>init</strong>()<br>
from functools import partial</p>
<pre><code>    model = mobilenetv2(pretrained)
    self.features = model.features[:-1]

    self.total_idx = len(self.features)
    self.down_idx = [2, 4, 7, 14]

    if downsample_factor == 8:
        for i in range(self.down_idx[-2], self.down_idx[-1]):
            self.features[i].apply(
                partial(self._nostride_dilate, dilate=2)
            )
        for i in range(self.down_idx[-1], self.total_idx):
            self.features[i].apply(
                partial(self._nostride_dilate, dilate=4)
            )
    elif downsample_factor == 16:
        for i in range(self.down_idx[-1], self.total_idx):
            self.features[i].apply(
                partial(self._nostride_dilate, dilate=2)
            )

def _nostride_dilate(self, m, dilate):
    classname = m.__class__.__name__
    if classname.find('Conv') != -1:
        if m.stride == (2, 2):
            m.stride = (1, 1)
            if m.kernel_size == (3, 3):
                m.dilation = (dilate // 2, dilate // 2)
                m.padding = (dilate // 2, dilate // 2)
        else:
            if m.kernel_size == (3, 3):
                m.dilation = (dilate, dilate)
                m.padding = (dilate, dilate)

def forward(self, x):
    low_level_features = x
    for i in range(4):
        low_level_features = self.features[i](low_level_features)
    x = low_level_features
    for i in range(4, len(self.features)):
        x = self.features[i](x)
    return low_level_features, x
</code></pre>
<p>class ASPP(nn.Module):<br>
def <strong>init</strong>(self, dim_in, dim_out, rate=1, bn_mom=0.1):<br>
super(ASPP, self).<strong>init</strong>()<br>
self.branch1 = nn.Sequential(<br>
nn.Conv2d(dim_in, dim_out, 1, 1, padding=0, dilation=rate, bias=True),<br>
nn.BatchNorm2d(dim_out, momentum=bn_mom),<br>
nn.ReLU(inplace=True),<br>
)<br>
self.branch2 = nn.Sequential(<br>
nn.Conv2d(dim_in, dim_out, (1, 3), padding=(0, 3 * rate), dilation=rate, bias=True),<br>
nn.Conv2d(dim_out, dim_out, (3, 1), padding=(3 * rate, 0), dilation=rate, bias=True),<br>
nn.BatchNorm2d(dim_out, momentum=bn_mom),<br>
nn.ReLU(inplace=True),<br>
)<br>
self.branch3 = nn.Sequential(<br>
nn.Conv2d(dim_in, dim_out, (1, 5), padding=(0, 5 * rate), dilation=rate, bias=True),<br>
nn.Conv2d(dim_out, dim_out, (5, 1), padding=(5 * rate, 0), dilation=rate, bias=True),<br>
nn.BatchNorm2d(dim_out, momentum=bn_mom),<br>
nn.ReLU(inplace=True),<br>
)<br>
self.branch4 = nn.Sequential(<br>
nn.Conv2d(dim_in, dim_out, (1, 7), padding=(0, 7 * rate), dilation=rate, bias=True),<br>
nn.Conv2d(dim_out, dim_out, (7, 1), padding=(7 * rate, 0), dilation=rate, bias=True),<br>
nn.BatchNorm2d(dim_out, momentum=bn_mom),<br>
nn.ReLU(inplace=True),<br>
)<br>
self.branch5_conv = nn.Conv2d(dim_in, dim_out, 1, 1, 0, bias=True)<br>
self.branch5_bn = nn.BatchNorm2d(dim_out, momentum=bn_mom)<br>
self.branch5_relu = nn.ReLU(inplace=True)</p>
<pre><code>    self.conv_cat = nn.Sequential(
        nn.Conv2d(dim_out * 5, dim_out, 1, 1, padding=0, bias=True),
        nn.BatchNorm2d(dim_out, momentum=bn_mom),
        nn.ReLU(inplace=True),
    )

def forward(self, x):
    [b, c, row, col] = x.size()

    conv1x1 = self.branch1(x)
    conv3x3_1 = self.branch2(x)
    conv3x3_2 = self.branch3(x)
    conv3x3_3 = self.branch4(x)

    global_feature = torch.mean(x, 2, True)
    global_feature = torch.mean(global_feature, 3, True)
    global_feature = self.branch5_conv(global_feature)
    global_feature = self.branch5_bn(global_feature)
    global_feature = self.branch5_relu(global_feature)
    global_feature = F.interpolate(global_feature, (row, col), mode='bilinear', align_corners=True)

 
    print(f"conv1x1 size: {conv1x1.size()}")
    print(f"conv3x3_1 size: {conv3x3_1.size()}")
    print(f"conv3x3_2 size: {conv3x3_2.size()}")
    print(f"conv3x3_3 size: {conv3x3_3.size()}")
    print(f"global_feature size: {global_feature.size()}")

    assert conv1x1.size() == (b, 256, row, col), f"conv1x1 size mismatch: {conv1x1.size()}"
    assert conv3x3_1.size() == (b, 256, row, col), f"conv3x3_1 size mismatch: {conv3x3_1.size()}"
    assert conv3x3_2.size() == (b, 256, row, col), f"conv3x3_2 size mismatch: {conv3x3_2.size()}"
    assert conv3x3_3.size() == (b, 256, row, col), f"conv3x3_3 size mismatch: {conv3x3_3.size()}"
    assert global_feature.size() == (b, 256, row, col), f"global_feature size mismatch: {global_feature.size()}"

    feature_cat = torch.cat([conv1x1, conv3x3_1, conv3x3_2, conv3x3_3, global_feature], dim=1)
    result = self.conv_cat(feature_cat)
    return result
</code></pre>
<p>class DeepLab(nn.Module):<br>
def <strong>init</strong>(self, num_classes, backbone=“mobilenet”, pretrained=True, downsample_factor=16):<br>
super(DeepLab, self).<strong>init</strong>()<br>
if backbone == “xception”:<br>
self.backbone = xception(downsample_factor=downsample_factor, pretrained=pretrained)<br>
in_channels = 2048<br>
low_level_channels = 256<br>
elif backbone == “mobilenet”:<br>
self.backbone = MobileNetV2(downsample_factor=downsample_factor, pretrained=pretrained)<br>
in_channels = 320<br>
low_level_channels = 24<br>
else:<br>
raise ValueError(‘Unsupported backbone - <code>{}</code>, Use mobilenet, xception.’.format(backbone))</p>
<pre><code>    self.aspp = ASPP(dim_in=in_channels, dim_out=256, rate=16 // downsample_factor)

    self.shortcut_conv = nn.Sequential(
        nn.Conv2d(low_level_channels, 48, 1),
        nn.BatchNorm2d(48),
        nn.ReLU(inplace=True)
    )

    self.cat_conv = nn.Sequential(
        nn.Conv2d(48 + 256, 256, 3, stride=1, padding=1),
        nn.BatchNorm2d(256),
        nn.ReLU(inplace=True),
        nn.Dropout(0.5),

        nn.Conv2d(256, 256, 3, stride=1, padding=1),
        nn.BatchNorm2d(256),
        nn.ReLU(inplace=True),

        nn.Dropout(0.1),
    )
    self.cls_conv = nn.Conv2d(256, num_classes, 1, stride=1)

def forward(self, x):
    H, W = x.size(2), x.size(3)

    low_level_features, x = self.backbone(x)
    x = self.aspp(x)
    low_level_features = self.shortcut_conv(low_level_features)

    x = F.interpolate(x, size=(low_level_features.size(2), low_level_features.size(3)), mode='bilinear',
                      align_corners=True)
    assert x.size(2) == low_level_features.size(2) and x.size(3) == low_level_features.size(3), \
        f"Size mismatch: {x.size()} vs {low_level_features.size()}"

    x = self.cat_conv(torch.cat((x, low_level_features), dim=1))
    x = self.cls_conv(x)
    x = F.interpolate(x, size=(H, W), mode='bilinear', align_corners=True)
    return x
</code></pre>
<p>AssertionError: conv3x3_1 size mismatch: torch.Size([8, 256, 72, 72])</p>

ptrblck: <p>The assert seems to be raised in your custom code, so could you check where it’s failing and what this assert statement checks?</p>

lililila: <p>Thanks a lot for your reply！This is where the error was reported：<br>
line 117, in forward<br>
assert conv3x3_1.size() == (b, 256, row, col), f"conv3x3_1 size mismatch: {conv3x3_1.size()}"</p>
<p>And This assertion statement is checking the size of the output of each branch：<br>
conv3x3_1 size: torch.Size([8, 256, 72, 72])<br>
conv3x3_2 size: torch.Size([8, 256, 76, 76])<br>
conv3x3_3 size: torch.Size([8, 256, 80, 80])<br>
global_feature size: torch.Size([8, 256, 64, 64])</p>
<p>The problem should be that replacing the dilated convolution with an asymmetric convolution is that the output size of each branch in the ASPP is not as expected. Unfortunately I don’t have enough programming skills to solve this problem.</p>