Repository: open-mmlab/mmaction2

Issue:
TSN temporal stream
Hello
I am trying to create a temporal stream for TSN using optical flow. Given the configuration below, I get the following error: `RuntimeError: Given groups=1, weight of size [64, 3, 7, 7], expected input[3, 10, 224, 224] to have 3 channels, but got 10 channels instead`.
The number of channels in the input after going through the training pipeline doesn't match the expected input of the first convolutional layer of the Resnet backbone model.
If I attempt to change the number of channels the backbone model expects, it leads to a failure when the model attempts to load the weights from torchvision.
What can be done so that I can train a TSN temporal stream?

You can find my Python configuration file as a text file here [tsn_r50_1x1x3_2e_ucf101_flow.py.txt](https://github.com/open-mmlab/mmaction2/files/5874657/tsn_r50_1x1x3_2e_ucf101_flow.py.txt)

TSN temporal stream
Hello
I am trying to create a temporal stream for TSN using optical flow. Given the configuration below, I get the following error: `RuntimeError: Given groups=1, weight of size [64, 3, 7, 7], expected input[3, 10, 224, 224] to have 3 channels, but got 10 channels instead`.
The number of channels in the input after going through the training pipeline doesn't match the expected input of the first convolutional layer of the Resnet backbone model.
If I attempt to change the number of channels the backbone model expects, it leads to a failure when the model attempts to load the weights from torchvision.
What can be done so that I can train a TSN temporal stream?

You can find my Python configuration file as a text file here [tsn_r50_1x1x3_2e_ucf101_flow.py.txt](https://github.com/open-mmlab/mmaction2/files/5874657/tsn_r50_1x1x3_2e_ucf101_flow.py.txt)



Discussion:

Resolving patch:
diff --git a/mmaction/models/backbones/resnet.py b/mmaction/models/backbones/resnet.py
--- a/mmaction/models/backbones/resnet.py
+++ b/mmaction/models/backbones/resnet.py
@@ -432,13 +432,15 @@ def _load_conv_params(conv, state_dict_tv, module_name_tv,
         """
 
         weight_tv_name = module_name_tv + '.weight'
-        conv.weight.data.copy_(state_dict_tv[weight_tv_name])
-        loaded_param_names.append(weight_tv_name)
+        if conv.weight.data.shape == state_dict_tv[weight_tv_name].shape:
+            conv.weight.data.copy_(state_dict_tv[weight_tv_name])
+            loaded_param_names.append(weight_tv_name)
 
         if getattr(conv, 'bias') is not None:
             bias_tv_name = module_name_tv + '.bias'
-            conv.bias.data.copy_(state_dict_tv[bias_tv_name])
-            loaded_param_names.append(bias_tv_name)
+            if conv.bias.data.shape == state_dict_tv[bias_tv_name].shape:
+                conv.bias.data.copy_(state_dict_tv[bias_tv_name])
+                loaded_param_names.append(bias_tv_name)
 
     @staticmethod
     def _load_bn_params(bn, state_dict_tv, module_name_tv, loaded_param_names):
@@ -457,16 +459,18 @@ def _load_bn_params(bn, state_dict_tv, module_name_tv, loaded_param_names):
         for param_name, param in bn.named_parameters():
             param_tv_name = f'{module_name_tv}.{param_name}'
             param_tv = state_dict_tv[param_tv_name]
-            param.data.copy_(param_tv)
-            loaded_param_names.append(param_tv_name)
+            if param.data.shape == param_tv.shape:
+                param.data.copy_(param_tv)
+                loaded_param_names.append(param_tv_name)
 
         for param_name, param in bn.named_buffers():
             param_tv_name = f'{module_name_tv}.{param_name}'
             # some buffers like num_batches_tracked may not exist
             if param_tv_name in state_dict_tv:
                 param_tv = state_dict_tv[param_tv_name]
-                param.data.copy_(param_tv)
-                loaded_param_names.append(param_tv_name)
+                if param.data.shape == param_tv.shape:
+                    param.data.copy_(param_tv)
+                    loaded_param_names.append(param_tv_name)
 
     def _load_torchvision_checkpoint(self, logger=None):
         """Initiate the parameters from torchvision pretrained checkpoint."""


Test patch:
diff --git a/tests/test_models/test_backbones.py b/tests/test_models/test_backbones.py
--- a/tests/test_models/test_backbones.py
+++ b/tests/test_models/test_backbones.py
@@ -13,7 +13,7 @@
 
 
 def test_x3d_backbone():
-    """Test resnet3d backbone."""
+    """Test x3d backbone."""
     with pytest.raises(AssertionError):
         # In X3D: 1 <= num_stages <= 4
         X3D(gamma_w=1.0, gamma_b=2.25, gamma_d=2.2, num_stages=0)
@@ -100,7 +100,7 @@ def test_x3d_backbone():
                                torch.zeros_like(m.conv3.bn.bias))
 
     # x3d_s inference
-    input_shape = (1, 3, 13, 160, 160)
+    input_shape = (1, 3, 13, 64, 64)
     imgs = generate_backbone_demo_inputs(input_shape)
     # parrots 3dconv is only implemented on gpu
     if torch.__version__ == 'parrots':
@@ -108,13 +108,13 @@ def test_x3d_backbone():
             x3d_s_frozen = x3d_s_frozen.cuda()
             imgs_gpu = imgs.cuda()
             feat = x3d_s_frozen(imgs_gpu)
-            assert feat.shape == torch.Size([1, 432, 13, 5, 5])
+            assert feat.shape == torch.Size([1, 432, 13, 2, 2])
     else:
         feat = x3d_s_frozen(imgs)
-        assert feat.shape == torch.Size([1, 432, 13, 5, 5])
+        assert feat.shape == torch.Size([1, 432, 13, 2, 2])
 
     # x3d_m inference
-    input_shape = (1, 3, 16, 224, 224)
+    input_shape = (1, 3, 16, 96, 96)
     imgs = generate_backbone_demo_inputs(input_shape)
     # parrots 3dconv is only implemented on gpu
     if torch.__version__ == 'parrots':
@@ -122,10 +122,10 @@ def test_x3d_backbone():
             x3d_s_frozen = x3d_s_frozen.cuda()
             imgs_gpu = imgs.cuda()
             feat = x3d_s_frozen(imgs_gpu)
-            assert feat.shape == torch.Size([1, 432, 16, 7, 7])
+            assert feat.shape == torch.Size([1, 432, 16, 3, 3])
     else:
         feat = x3d_s_frozen(imgs)
-        assert feat.shape == torch.Size([1, 432, 16, 7, 7])
+        assert feat.shape == torch.Size([1, 432, 16, 3, 3])
 
 
 def test_resnet2plus1d_backbone():
diff --git a/tests/test_models/test_common_modules/test_resnet.py b/tests/test_models/test_common_modules/test_resnet.py
--- a/tests/test_models/test_common_modules/test_resnet.py
+++ b/tests/test_models/test_common_modules/test_resnet.py
@@ -111,3 +111,17 @@ def test_resnet_backbone():
     resnet50_caffe.train()
     feat = resnet50_caffe(imgs)
     assert feat.shape == torch.Size([1, 2048, 2, 2])
+
+    resnet50_flow = ResNet(
+        depth=50, pretrained='torchvision://resnet50', in_channels=10)
+    input_shape = (1, 10, 64, 64)
+    imgs = generate_backbone_demo_inputs(input_shape)
+    feat = resnet50_flow(imgs)
+    assert feat.shape == torch.Size([1, 2048, 2, 2])
+
+    resnet50 = ResNet(
+        depth=50, pretrained='torchvision://resnet50', in_channels=3)
+    input_shape = (1, 3, 64, 64)
+    imgs = generate_backbone_demo_inputs(input_shape)
+    feat = resnet50(imgs)
+    assert feat.shape == torch.Size([1, 2048, 2, 2])