Repository: open-mmlab/mmpretrain

Issue:
Conformer model does not support different input image sizes
The conformer model backbone gives error when tries to give input image size like (241x241) or (242x242) ,but works fine with some image sizes and also changes in  hyperparameters patch_size gives the error in tensor size matching . It is better if it has hyperparameter like img_size like other models



Discussion:
Sounds like a bug, we will fix it later
> Sounds like a bug, we will fix it later

Thanks

Resolving patch:
diff --git a/mmcls/models/backbones/conformer.py b/mmcls/models/backbones/conformer.py
--- a/mmcls/models/backbones/conformer.py
+++ b/mmcls/models/backbones/conformer.py
@@ -5,6 +5,7 @@
 import torch.nn.functional as F
 from mmcv.cnn import build_activation_layer, build_norm_layer
 from mmcv.cnn.bricks.drop import DropPath
+from mmcv.cnn.bricks.transformer import AdaptivePadding
 from mmcv.cnn.utils.weight_init import trunc_normal_
 
 from mmcls.utils import get_root_logger
@@ -438,9 +439,16 @@ def __init__(self,
         self.maxpool = nn.MaxPool2d(
             kernel_size=3, stride=2, padding=1)  # 1 / 4 [56, 56]
 
+        assert patch_size % 16 == 0, 'The patch size of Conformer must ' \
+            'be divisible by 16.'
+        trans_down_stride = patch_size // 4
+
+        # To solve the issue #680
+        # Auto pad the feature map to be divisible by trans_down_stride
+        self.auto_pad = AdaptivePadding(trans_down_stride, trans_down_stride)
+
         # 1 stage
         stage1_channels = int(base_channels * self.channel_ratio)
-        trans_down_stride = patch_size // 4
         self.conv_1 = ConvBlock(
             in_channels=64,
             out_channels=stage1_channels,
@@ -587,6 +595,7 @@ def forward(self, x):
 
         # stem
         x_base = self.maxpool(self.act1(self.bn1(self.conv1(x))))
+        x_base = self.auto_pad(x_base)
 
         # 1 stage [N, 64, 56, 56] -> [N, 128, 56, 56]
         x = self.conv_1(x_base, out_conv2=False)


Test patch:
diff --git a/tests/test_models/test_backbones/test_conformer.py b/tests/test_models/test_backbones/test_conformer.py
--- a/tests/test_models/test_backbones/test_conformer.py
+++ b/tests/test_models/test_backbones/test_conformer.py
@@ -57,6 +57,25 @@ def test_conformer_backbone():
                                   )  # base_channels * channel_ratio * 4
     assert transformer_feature.shape == (3, 384)
 
+    # Test Conformer with irregular input size.
+    model = Conformer(**cfg_ori)
+    model.init_weights()
+    model.train()
+
+    assert check_norm_state(model.modules(), True)
+
+    imgs = torch.randn(3, 3, 241, 241)
+    conv_feature, transformer_feature = model(imgs)[-1]
+    assert conv_feature.shape == (3, 64 * 1 * 4
+                                  )  # base_channels * channel_ratio * 4
+    assert transformer_feature.shape == (3, 384)
+
+    imgs = torch.randn(3, 3, 321, 221)
+    conv_feature, transformer_feature = model(imgs)[-1]
+    assert conv_feature.shape == (3, 64 * 1 * 4
+                                  )  # base_channels * channel_ratio * 4
+    assert transformer_feature.shape == (3, 384)
+
     # Test custom arch Conformer without output cls token
     cfg = deepcopy(cfg_ori)
     cfg['arch'] = {
@@ -72,7 +91,7 @@ def test_conformer_backbone():
     assert conv_feature.shape == (3, 32 * 3 * 4)
     assert transformer_feature.shape == (3, 128)
 
-    # Test ViT with multi out indices
+    # Test Conformer with multi out indices
     cfg = deepcopy(cfg_ori)
     cfg['out_indices'] = [4, 8, 12]
     model = Conformer(**cfg)