Repository: open-mmlab/mmrazor (language: python)

Issue:
Knowledge Distillation for mmdetection model
Please guide about this problem

#### config file 

_base_ = [
    '../../_base_/datasets/mmdet/coco_detection.py',
    '../../_base_/schedules/mmdet/schedule_1x.py',
    '../../_base_/mmdet_runtime.py'
]


# model settings
student = dict(
    type='mmdet.DeformableDETR',
    backbone=dict(
        type='ResNet',
        depth=50,
        num_stages=4,
        out_indices=(0, 1, 2, 3),
        frozen_stages=-1,
        norm_cfg=dict(type='BN', requires_grad=True),
        norm_eval=True,
        style='pytorch',
        init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet50')),
    neck=dict(
        type='ChannelMapper',
        in_channels=[512, 1024, 2048],
        kernel_size=1,
        out_channels=256,
        act_cfg=None,
        norm_cfg=dict(type='GN', num_groups=32),
        num_outs=4),
    bbox_head=dict(
        type='DeformableDETRHead',
        num_query=300,
        num_classes=2,
        in_channels=2048,
        sync_cls_avg_factor=True,
        as_two_stage=True,
        transformer=dict(
            type='DeformableDetrTransformer',
            encoder=dict(
                type='DetrTransformerEncoder',
                num_layers=6,
                transformerlayers=dict(
                    type='BaseTransformerLayer',
                    attn_cfgs=dict(
                        type='MultiScaleDeformableAttention', embed_dims=256),
                    feedforward_channels=1024,
                    ffn_dropout=0.1,
                    operation_order=('self_attn', 'norm', 'ffn', 'norm'))),
            decoder=dict(
                type='DeformableDetrTransformerDecoder',
                num_layers=6,
                return_intermediate=True,
                transformerlayers=dict(
                    type='DetrTransformerDecoderLayer',
                    attn_cfgs=[
                        dict(
                            type='MultiheadAttention',
                            embed_dims=256,
                            num_heads=8,
                            dropout=0.1),
                        dict(
                            type='MultiScaleDeformableAttention',
                            embed_dims=256)
                    ],
                    feedforward_channels=1024,
                    ffn_dropout=0.1,
                    operation_order=('self_attn', 'norm', 'cross_attn', 'norm',
                                     'ffn', 'norm')))),
        positional_encoding=dict(
            type='SinePositionalEncoding',
            num_feats=128,
            normalize=True,
            offset=-0.5),
        loss_cls=dict(
            type='FocalLoss',
            use_sigmoid=True,
            gamma=2.0,
            alpha=0.25,
            loss_weight=2.0),
        loss_bbox=dict(type='L1Loss', loss_weight=5.0),
        loss_iou=dict(type='GIoULoss', loss_weight=2.0),
        with_box_refine=True),
    train_cfg=dict(
        assigner=dict(
            type='HungarianAssigner',
            cls_cost=dict(type='FocalLossCost', weight=2.0),
            reg_cost=dict(type='BBoxL1Cost', weight=5.0, box_format='xywh'),
            iou_cost=dict(type='IoUCost', iou_mode='giou', weight=2.0))),
    test_cfg=dict(max_per_img=100))

teacher = dict(
    type='mmdet.DeformableDETR',
    backbone=dict(
        type='ResNeXt',
        depth=101,
        groups=64,
        base_width=4,
        num_stages=4,
        out_indices=(1, 2, 3),
        frozen_stages=1,
        norm_cfg=dict(type='BN', requires_grad=True),
        style='pytorch',
        init_cfg=None),
    neck=dict(
        type='ChannelMapper',
        in_channels=[512, 1024, 2048],
        kernel_size=1,
        out_channels=256,
        act_cfg=None,
        norm_cfg=dict(type='GN', num_groups=32),
        num_outs=4),
    bbox_head=dict(
        type='DeformableDETRHead',
        num_query=300,
        num_classes=2,
        in_channels=2048,
        sync_cls_avg_factor=True,
        as_two_stage=True,
        transformer=dict(
            type='DeformableDetrTransformer',
            encoder=dict(
                type='DetrTransformerEncoder',
                num_layers=6,
                transformerlayers=dict(
                    type='BaseTransformerLayer',
                    attn_cfgs=dict(
                        type='MultiScaleDeformableAttention', embed_dims=256),
                    feedforward_channels=1024,
                    ffn_dropout=0.1,
                    operation_order=('self_attn', 'norm', 'ffn', 'norm'))),
            decoder=dict(
                type='DeformableDetrTransformerDecoder',
                num_layers=6,
                return_intermediate=True,
                transformerlayers=dict(
                    type='DetrTransformerDecoderLayer',
                    attn_cfgs=[
                        dict(
                            type='MultiheadAttention',
                            embed_dims=256,
                            num_heads=8,
                            dropout=0.1),
                        dict(
                            type='MultiScaleDeformableAttention',
                            embed_dims=256)
                    ],
                    feedforward_channels=1024,
                    ffn_dropout=0.1,
                    operation_order=('self_attn', 'norm', 'cross_attn', 'norm',
                                     'ffn', 'norm')))),
        positional_encoding=dict(
            type='SinePositionalEncoding',
            num_feats=128,
            normalize=True,
            offset=-0.5),
        loss_cls=dict(
            type='FocalLoss',
            use_sigmoid=True,
            gamma=2.0,
            alpha=0.25,
            loss_weight=2.0),
        loss_bbox=dict(type='L1Loss', loss_weight=5.0),
        loss_iou=dict(type='GIoULoss', loss_weight=2.0),
        with_box_refine=True),
    train_cfg=dict(
        assigner=dict(
            type='HungarianAssigner',
            cls_cost=dict(type='FocalLossCost', weight=2.0),
            reg_cost=dict(type='BBoxL1Cost', weight=5.0, box_format='xywh'),
            iou_cost=dict(type='IoUCost', iou_mode='giou', weight=2.0))),
    test_cfg=dict(max_per_img=100))




img_norm_cfg = dict(
    mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True)
train_pipeline = [
    dict(type='LoadImageFromFile'),
    dict(type='LoadAnnotations', with_bbox=True),
    dict(type='RandomFlip', flip_ratio=0.5),
    dict(
        type='AutoAugment',
        policies=[[{
            'type':
            'Resize',
            'img_scale': [(480, 800), (512, 800), (544, 800), (576, 800),
                          (608, 800), (640, 800), (672, 800), (704, 800),
                          (736, 800), (768, 800), (800, 800)],
            'multiscale_mode':
            'value',
            'keep_ratio':
            True
        }],
                  [{
                      'type': 'Resize',
                      'img_scale': [(400, 4200), (500, 4200), (600, 4200)],
                      'multiscale_mode': 'value',
                      'keep_ratio': True
                  }, {
                      'type': 'RandomCrop',
                      'crop_type': 'absolute_range',
                      'crop_size': (384, 600),
                      'allow_negative_crop': True
                  }, {
                      'type':
                      'Resize',
                      'img_scale': [(480, 800), (512, 800), (544, 800),
                                    (576, 800), (608, 800), (640, 800),
                                    (672, 800), (704, 800), (736, 800),
                                    (768, 800), (800, 800)],
                      'multiscale_mode':
                      'value',
                      'override':
                      True,
                      'keep_ratio':
                      True
                  }]]),
    dict(
        type='Normalize',
        mean=[123.675, 116.28, 103.53],
        std=[58.395, 57.12, 57.375],
        to_rgb=True),
    dict(type='Pad', size_divisor=1),
    dict(type='DefaultFormatBundle'),
    dict(type='Collect', keys=['img', 'gt_bboxes', 'gt_labels'])
]
test_pipeline = [
    dict(type='LoadImageFromFile'),
    dict(
        type='MultiScaleFlipAug',
        img_scale=(800, 800),
        flip=False,
        transforms=[
            dict(type='Resize', keep_ratio=True),
            dict(type='RandomFlip'),
            dict(
                type='Normalize',
                mean=[123.675, 116.28, 103.53],
                std=[58.395, 57.12, 57.375],
                to_rgb=True),
            dict(type='Pad', size_divisor=1),
            dict(type='ImageToTensor', keys=['img']),
            dict(type='Collect', keys=['img'])
        ])
]


optimizer = dict(
    type='AdamW',
    lr=0.001,
    weight_decay=0.0001,
    paramwise_cfg=dict(
        custom_keys=dict(
            backbone=dict(lr_mult=0.1),
            sampling_offsets=dict(lr_mult=0.1),
            reference_points=dict(lr_mult=0.1))))
optimizer_config = dict(grad_clip=dict(max_norm=0.1, norm_type=2))
lr_config = dict(policy='step', gamma=0.5, step=[40, 80, 120, 160, 200])
runner = dict(type='EpochBasedRunner', max_epochs=300)
evaluation= dict(classwise=True, interval=1, metric='bbox')
checkpoint_config = dict(interval=1)

log_config = dict(
    interval=50,
    hooks=[dict(type='TextLoggerHook')])

custom_hooks = [dict(type='NumClassCheckHook')]
dist_params = dict(backend='nccl')
log_level = 'INFO'
load_from = None
resume_from = './teacher_model/detr/deformable_detr_epoch_126.pth'
workflow = [('train', 1)]


# algorithm setting
algorithm = dict(
    type='GeneralDistill',
    architecture=dict(
        type='MMDetArchitecture',
        model=student,
    ),
    distiller=dict(
        type='SingleTeacherDistiller',
        teacher=teacher,
        teacher_trainable=False,
        components=[
            dict(
                student_module='bbox_head.cls_branches',
                teacher_module='bbox_head.cls_branches',
                losses=[
                    dict(
                        type='ChannelWiseDivergence',
                        name='loss_cwd_cls_head',
                        tau=1,
                        loss_weight=5,
                    )
                ])
        ]),
)

find_unused_parameters = True
########### mmrazor execution

CUDA_VISIBLE_DEVICES=${gpus} python tools/mmdet/train_mmdet.py ./configs/distill/cwd/cwd_cls_head_deformable_detr_rx101_r50.py --cfg-options algorithm.distiller.teacher.init_cfg.type=Pretrained algorithm.distiller.teacher.init_cfg.checkpoint=./teacher_model/detr/deformable_detr_epoch_126.pth

############ error message
Traceback (most recent call last):
  File "tools/mmdet/train_mmdet.py", line 204, in <module>
    main()
  File "tools/mmdet/train_mmdet.py", line 200, in main
    meta=meta)
  File "/home/jovyan/nas/1_user/dhkyung@agilesoda.ai/checkbox_gen/mmrazor/mmrazor/apis/mmdet/train.py", line 192, in train_detector
    runner.resume(cfg.resume_from)
  File "/opt/conda/envs/mmrazor/lib/python3.7/site-packages/mmcv/runner/base_runner.py", line 389, in resume
    self.optimizer.load_state_dict(checkpoint['optimizer'])
  File "/opt/conda/envs/mmrazor/lib/python3.7/site-packages/torch/optim/optimizer.py", line 119, in load_state_dict
    raise ValueError("loaded state dict has a different number of "
ValueError: loaded state dict has a different number of parameter groups




Discussion / hints:
Your resume ckpt seems to be the same as the teacher's ckpt.
The resume ckpt must be  trained with MMRazor.
Thank you for your comment.

So, I removed my resume and solved this error.

But, another error is occurred...

Traceback (most recent call last):
  File "tools/mmdet/train_mmdet.py", line 204, in <module>
    main()
  File "tools/mmdet/train_mmdet.py", line 200, in main
    meta=meta)
  File "/home/...../mmrazor/mmrazor/apis/mmdet/train.py", line 195, in train_detector
    runner.run(data_loader, cfg.workflow)
  File "/opt/conda/envs/mmrazor/lib/python3.7/site-packages/mmcv/runner/epoch_based_runner.py", line 127, in run
    epoch_runner(data_loaders[i], **kwargs)
  File "/opt/conda/envs/mmrazor/lib/python3.7/site-packages/mmcv/runner/epoch_based_runner.py", line 50, in train
    self.run_iter(data_batch, train_mode=True, **kwargs)
  File "/opt/conda/envs/mmrazor/lib/python3.7/site-packages/mmcv/runner/epoch_based_runner.py", line 30, in run_iter
    **kwargs)
  File "/opt/conda/envs/mmrazor/lib/python3.7/site-packages/mmcv/parallel/data_parallel.py", line 75, in train_step
    return self.module.train_step(*inputs[0], **kwargs[0])
  File "/home/...../mmrazor/mmrazor/models/algorithms/general_distill.py", line 41, in train_step
    self.architecture, data)
  File "/home/...../mmrazor/mmrazor/models/distillers/single_teacher.py", line 191, in exec_student_forward
    output = student(**data)
  File "/opt/conda/envs/mmrazor/lib/python3.7/site-packages/torch/nn/modules/module.py", line 727, in _call_impl
    result = self.forward(*input, **kwargs)
  File "/home/...../mmrazor/mmrazor/models/architectures/base.py", line 33, in forward
    return self.model(img, return_loss=return_loss, **kwargs)
  File "/opt/conda/envs/mmrazor/lib/python3.7/site-packages/torch/nn/modules/module.py", line 727, in _call_impl
    result = self.forward(*input, **kwargs)
  File "/opt/conda/envs/mmrazor/lib/python3.7/site-packages/mmcv/runner/fp16_utils.py", line 98, in new_func
    return old_func(*args, **kwargs)
  File "/home/...../mmdetection/mmdet/models/detectors/base.py", line 172, in forward
    return self.forward_train(img, img_metas, **kwargs)
  File "/home/...../mmdetection/mmdet/models/detectors/single_stage.py", line 82, in forward_train
    x = self.extract_feat(img)
  File "/home/...../mmdetection/mmdet/models/detectors/single_stage.py", line 45, in extract_feat
    x = self.neck(x)
  File "/opt/conda/envs/mmrazor/lib/python3.7/site-packages/torch/nn/modules/module.py", line 727, in _call_impl
    result = self.forward(*input, **kwargs)
  File "/home/...../mmdetection/mmdet/models/necks/channel_mapper.py", line 92, in forward
    assert len(inputs) == len(self.convs)
AssertionError



my setting is

sys.platform: linux
Python: 3.7.12 | packaged by conda-forge | (default, Oct 26 2021, 06:08:21) [GCC 9.4.0]
CUDA available: True
GPU 0: Tesla V100-DGXS-32GB
CUDA_HOME: /usr/local/cuda
NVCC: Build cuda_11.0_bu.TC445_37.28845127_0
GCC: gcc (Ubuntu 7.5.0-3ubuntu1~18.04) 7.5.0
PyTorch: 1.7.0
PyTorch compiling details: PyTorch built with:
  - GCC 7.3
  - C++ Version: 201402
  - Intel(R) Math Kernel Library Version 2020.0.4 Product Build 20200917 for Intel(R) 64 architecture applications
  - Intel(R) MKL-DNN v1.6.0 (Git Hash 5ef631a030a6f73131c77892041042805a06064f)
  - OpenMP 201511 (a.k.a. OpenMP 4.5)
  - NNPACK is enabled
  - CPU capability usage: AVX2
  - CUDA Runtime 11.0
  - NVCC architecture flags: -gencode;arch=compute_37,code=sm_37;-gencode;arch=compute_50,code=sm_50;-gencode;arch=compute_60,code=sm_60;-gencode;arch=compute_61,code=sm_61;-gencode;arch=compute_70,code=sm_70;-gencode;arch=compute_75,code=sm_75;-gencode;arch=compute_80,code=sm_80;-gencode;arch=compute_37,code=compute_37
  - CuDNN 8.0.3
  - Magma 2.5.2
  - Build settings: BLAS=MKL, BUILD_TYPE=Release, CXX_FLAGS= -Wno-deprecated -fvisibility-inlines-hidden -DUSE_PTHREADPOOL -fopenmp -DNDEBUG -DUSE_FBGEMM -DUSE_QNNPACK -DUSE_PYTORCH_QNNPACK -DUSE_XNNPACK -DUSE_VULKAN_WRAPPER -O2 -fPIC -Wno-narrowing -Wall -Wextra -Werror=return-type -Wno-missing-field-initializers -Wno-type-limits -Wno-array-bounds -Wno-unknown-pragmas -Wno-sign-compare -Wno-unused-parameter -Wno-unused-variable -Wno-unused-function -Wno-unused-result -Wno-unused-local-typedefs -Wno-strict-overflow -Wno-strict-aliasing -Wno-error=deprecated-declarations -Wno-stringop-overflow -Wno-psabi -Wno-error=pedantic -Wno-error=redundant-decls -Wno-error=old-style-cast -fdiagnostics-color=always -faligned-new -Wno-unused-but-set-variable -Wno-maybe-uninitialized -fno-math-errno -fno-trapping-math -Werror=format -Wno-stringop-overflow, PERF_WITH_AVX=1, PERF_WITH_AVX2=1, PERF_WITH_AVX512=1, USE_CUDA=ON, USE_EXCEPTION_PTR=1, USE_GFLAGS=OFF, USE_GLOG=OFF, USE_MKL=ON, USE_MKLDNN=ON, USE_MPI=OFF, USE_NCCL=ON, USE_NNPACK=ON, USE_OPENMP=ON, 

TorchVision: 0.8.0
OpenCV: 4.5.5
MMCV: 1.4.2
MMCV Compiler: GCC 7.3
MMCV CUDA Compiler: 11.0
MMDetection: 2.22.0+81e0e34
mmrazor: 0.2.0 

how can i find solution ??
In this setting, I can running code of learning mmdetection model
It seems something wrong in your detector config.
In backbone your set `out_indices=(0, 1, 2, 3)`, but in neck you set `in_channels=[512, 1024, 2048]`.
The two tuples' lengths should be the same.
After editting, it is running :)

Thank you very much

Fix patch:
diff --git a/configs/distill/mmdet/mgd/README.md b/configs/distill/mmdet/mgd/README.md
new file mode 100644
index 000000000..19c52ac4b
--- /dev/null
+++ b/configs/distill/mmdet/mgd/README.md
@@ -0,0 +1,30 @@
+# MGD
+
+> [Masked Generative Distillation](https://arxiv.org/abs/2205.01529)
+
+<!-- [ALGORITHM] -->
+
+## Abstract
+
+Knowledge distillation has been applied to various tasks successfully. The current distillation algorithm usually improves students' performance by imitating the output of the teacher. This paper shows that teachers can also improve students' representation power by guiding students' feature recovery. From this point of view, we propose Masked Generative Distillation (MGD), which is simple: we mask random pixels of the student's feature and force it to generate the teacher's full feature through a simple block. MGD is a truly general feature-based distillation method, which can be utilized on various tasks, including image classification, object detection, semantic segmentation and instance segmentation. We experiment on different models with extensive datasets and the results show that all the students achieve excellent improvements. Notably, we boost ResNet-18 from 69.90% to 71.69% ImageNet top-1 accuracy, RetinaNet with ResNet-50 backbone from 37.4 to 41.0 Boundingbox mAP, SOLO based on ResNet-50 from 33.1 to 36.2 Mask mAP and DeepLabV3 based on ResNet-18 from 73.20 to 76.02 mIoU.
+
+![pipeline](https://github.com/yzd-v/MGD/raw/master/architecture.png)
+
+## Results and models
+
+### Detection
+
+| Location | Dataset |                                                            Teacher                                                             |                                                        Student                                                         | Lr schd | mAP  | mAP(T) | mAP(S) |                       Config                        | Download                                                                                                                                                                                                                                                                                                                                                                                                              |
+| :------: | :-----: | :----------------------------------------------------------------------------------------------------------------------------: | :--------------------------------------------------------------------------------------------------------------------: | :-----: | :--: | :----: | :----: | :-------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+|   FPN    |  COCO   | [RetinaNet-X101](https://github.com/open-mmlab/mmdetection/blob/dev-3.x/configs/retinanet/retinanet_x101-64x4d_fpn_1x_coco.py) | [RetinaNet-R50](https://github.com/open-mmlab/mmdetection/blob/dev-3.x/configs/retinanet/retinanet_r50_fpn_2x_coco.py) |   2x    | 41.0 |  41.0  |  37.4  | [config](mgd_fpn_retina_x101_retina_r50_2x_coco.py) | [teacher](https://download.openmmlab.com/mmdetection/v2.0/retinanet/retinanet_x101_64x4d_fpn_1x_coco/retinanet_x101_64x4d_fpn_1x_coco_20200130-366f5af1.pth) \|[model](https://download.openmmlab.com/mmrazor/v1/mgd/mgd_fpn_retina_x101_retina_r50_2x_coco_20221209_191847-87141529.pth) \| [log](https://download.openmmlab.com/mmrazor/v1/mgd/mgd_fpn_retina_x101_retina_r50_2x_coco_20221209_191847-87141529.log) |
+
+## Citation
+
+```latex
+@article{yang2022masked,
+  title={Masked Generative Distillation},
+  author={Yang, Zhendong and Li, Zhe and Shao, Mingqi and Shi, Dachuan and Yuan, Zehuan and Yuan, Chun},
+  journal={arXiv preprint arXiv:2205.01529},
+  year={2022}
+}
+```
diff --git a/configs/distill/mmdet/mgd/mgd_fpn_retina_x101_retina_r50_2x_coco.py b/configs/distill/mmdet/mgd/mgd_fpn_retina_x101_retina_r50_2x_coco.py
new file mode 100644
index 000000000..9a416cd8b
--- /dev/null
+++ b/configs/distill/mmdet/mgd/mgd_fpn_retina_x101_retina_r50_2x_coco.py
@@ -0,0 +1,118 @@
+_base_ = ['mmdet::retinanet/retinanet_r50_fpn_2x_coco.py']
+
+teacher_ckpt = 'https://download.openmmlab.com/mmdetection/v2.0/retinanet/retinanet_x101_64x4d_fpn_1x_coco/retinanet_x101_64x4d_fpn_1x_coco_20200130-366f5af1.pth'  # noqa: E501
+
+student = _base_.model
+student.neck.init_cfg = dict(
+    type='Pretrained', prefix='neck.', checkpoint=teacher_ckpt)
+student.bbox_head.init_cfg = dict(
+    type='Pretrained', prefix='bbox_head.', checkpoint=teacher_ckpt)
+
+model = dict(
+    _scope_='mmrazor',
+    _delete_=True,
+    type='FpnTeacherDistill',
+    architecture=student,
+    teacher=dict(
+        cfg_path='mmdet::retinanet/retinanet_x101-64x4d_fpn_1x_coco.py',
+        pretrained=False),
+    teacher_ckpt=teacher_ckpt,
+    distiller=dict(
+        type='ConfigurableDistiller',
+        student_recorders=dict(
+            fpn0=dict(type='ModuleOutputs', source='neck.fpn_convs.0.conv'),
+            fpn1=dict(type='ModuleOutputs', source='neck.fpn_convs.1.conv'),
+            fpn2=dict(type='ModuleOutputs', source='neck.fpn_convs.2.conv'),
+            fpn3=dict(type='ModuleOutputs', source='neck.fpn_convs.3.conv'),
+            fpn4=dict(type='ModuleOutputs', source='neck.fpn_convs.4.conv')),
+        teacher_recorders=dict(
+            fpn0=dict(type='ModuleOutputs', source='neck.fpn_convs.0.conv'),
+            fpn1=dict(type='ModuleOutputs', source='neck.fpn_convs.1.conv'),
+            fpn2=dict(type='ModuleOutputs', source='neck.fpn_convs.2.conv'),
+            fpn3=dict(type='ModuleOutputs', source='neck.fpn_convs.3.conv'),
+            fpn4=dict(type='ModuleOutputs', source='neck.fpn_convs.4.conv')),
+        connectors=dict(
+            s_fpn0_connector=dict(
+                type='MGDConnector',
+                student_channels=256,
+                teacher_channels=256,
+                lambda_mgd=0.65),
+            s_fpn1_connector=dict(
+                type='MGDConnector',
+                student_channels=256,
+                teacher_channels=256,
+                lambda_mgd=0.65),
+            s_fpn2_connector=dict(
+                type='MGDConnector',
+                student_channels=256,
+                teacher_channels=256,
+                lambda_mgd=0.65),
+            s_fpn3_connector=dict(
+                type='MGDConnector',
+                student_channels=256,
+                teacher_channels=256,
+                lambda_mgd=0.65),
+            s_fpn4_connector=dict(
+                type='MGDConnector',
+                student_channels=256,
+                teacher_channels=256,
+                lambda_mgd=0.65)),
+        distill_losses=dict(
+            loss_mgd_fpn0=dict(type='MGDLoss', alpha_mgd=0.00002),
+            loss_mgd_fpn1=dict(type='MGDLoss', alpha_mgd=0.00002),
+            loss_mgd_fpn2=dict(type='MGDLoss', alpha_mgd=0.00002),
+            loss_mgd_fpn3=dict(type='MGDLoss', alpha_mgd=0.00002),
+            loss_mgd_fpn4=dict(type='MGDLoss', alpha_mgd=0.00002)),
+        loss_forward_mappings=dict(
+            loss_mgd_fpn0=dict(
+                preds_S=dict(
+                    from_student=True,
+                    recorder='fpn0',
+                    connector='s_fpn0_connector'),
+                preds_T=dict(from_student=False, recorder='fpn0')),
+            loss_mgd_fpn1=dict(
+                preds_S=dict(
+                    from_student=True,
+                    recorder='fpn1',
+                    connector='s_fpn1_connector'),
+                preds_T=dict(from_student=False, recorder='fpn1')),
+            loss_mgd_fpn2=dict(
+                preds_S=dict(
+                    from_student=True,
+                    recorder='fpn2',
+                    connector='s_fpn2_connector'),
+                preds_T=dict(from_student=False, recorder='fpn2')),
+            loss_mgd_fpn3=dict(
+                preds_S=dict(
+                    from_student=True,
+                    recorder='fpn3',
+                    connector='s_fpn3_connector'),
+                preds_T=dict(from_student=False, recorder='fpn3')),
+            loss_mgd_fpn4=dict(
+                preds_S=dict(
+                    from_student=True,
+                    recorder='fpn4',
+                    connector='s_fpn4_connector'),
+                preds_T=dict(from_student=False, recorder='fpn4')))))
+
+find_unused_parameters = True
+
+val_cfg = dict(_delete_=True, type='mmrazor.SingleTeacherDistillValLoop')
+
+optimizer_config = dict(
+    _delete_=True, grad_clip=dict(max_norm=35, norm_type=2))
+
+param_scheduler = [
+    dict(
+        type='LinearLR', start_factor=0.001, by_epoch=False, begin=0, end=500),
+    dict(
+        type='MultiStepLR',
+        begin=0,
+        end=24,
+        by_epoch=True,
+        milestones=[16, 22],
+        gamma=0.1)
+]
+
+optim_wrapper = dict(
+    optimizer=dict(type='SGD', lr=0.01, momentum=0.9, weight_decay=0.0001))
diff --git a/mmrazor/models/architectures/connectors/__init__.py b/mmrazor/models/architectures/connectors/__init__.py
index c12aa60d7..b1dacebb8 100644
--- a/mmrazor/models/architectures/connectors/__init__.py
+++ b/mmrazor/models/architectures/connectors/__init__.py
@@ -4,11 +4,12 @@
 from .crd_connector import CRDConnector
 from .factor_transfer_connectors import Paraphraser, Translator
 from .fbkd_connector import FBKDStudentConnector, FBKDTeacherConnector
+from .mgd_connector import MGDConnector
 from .ofd_connector import OFDTeacherConnector
 from .torch_connector import TorchFunctionalConnector, TorchNNConnector
 
 __all__ = [
     'ConvModuleConncetor', 'Translator', 'Paraphraser', 'BYOTConnector',
     'FBKDTeacherConnector', 'FBKDStudentConnector', 'TorchFunctionalConnector',
-    'CRDConnector', 'TorchNNConnector', 'OFDTeacherConnector'
+    'CRDConnector', 'TorchNNConnector', 'OFDTeacherConnector', 'MGDConnector'
 ]
diff --git a/mmrazor/models/architectures/connectors/mgd_connector.py b/mmrazor/models/architectures/connectors/mgd_connector.py
new file mode 100644
index 000000000..41b77df4c
--- /dev/null
+++ b/mmrazor/models/architectures/connectors/mgd_connector.py
@@ -0,0 +1,65 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+from typing import Dict, Optional
+
+import torch
+import torch.nn as nn
+
+from mmrazor.registry import MODELS
+from .base_connector import BaseConnector
+
+
+@MODELS.register_module()
+class MGDConnector(BaseConnector):
+    """PyTorch version of `Masked Generative Distillation.
+
+    <https://arxiv.org/abs/2205.01529>`
+
+    Args:
+        student_channels(int): Number of channels in the student's feature map.
+        teacher_channels(int): Number of channels in the teacher's feature map.
+        lambda_mgd (float, optional): masked ratio. Defaults to 0.65
+        init_cfg (Optional[Dict], optional): The weight initialized config for
+            :class:`BaseModule`. Defaults to None.
+    """
+
+    def __init__(
+        self,
+        student_channels: int,
+        teacher_channels: int,
+        lambda_mgd: float = 0.65,
+        init_cfg: Optional[Dict] = None,
+    ) -> None:
+        super().__init__(init_cfg)
+        self.lambda_mgd = lambda_mgd
+        if student_channels != teacher_channels:
+            self.align = nn.Conv2d(
+                student_channels,
+                teacher_channels,
+                kernel_size=1,
+                stride=1,
+                padding=0)
+        else:
+            self.align = None
+
+        self.generation = nn.Sequential(
+            nn.Conv2d(
+                teacher_channels, teacher_channels, kernel_size=3, padding=1),
+            nn.ReLU(inplace=True),
+            nn.Conv2d(
+                teacher_channels, teacher_channels, kernel_size=3, padding=1))
+
+    def forward_train(self, feature: torch.Tensor) -> torch.Tensor:
+        if self.align is not None:
+            feature = self.align(feature)
+
+        N, C, H, W = feature.shape
+
+        device = feature.device
+        mat = torch.rand((N, 1, H, W)).to(device)
+        mat = torch.where(mat > 1 - self.lambda_mgd,
+                          torch.zeros(1).to(device),
+                          torch.ones(1).to(device)).to(device)
+
+        masked_fea = torch.mul(feature, mat)
+        new_fea = self.generation(masked_fea)
+        return new_fea
diff --git a/mmrazor/models/losses/__init__.py b/mmrazor/models/losses/__init__.py
index 39f82a91c..8c779d15a 100644
--- a/mmrazor/models/losses/__init__.py
+++ b/mmrazor/models/losses/__init__.py
@@ -12,6 +12,7 @@
 from .kl_divergence import KLDivergence
 from .l1_loss import L1Loss
 from .l2_loss import L2Loss
+from .mgd_loss import MGDLoss
 from .ofd_loss import OFDLoss
 from .pkd_loss import PKDLoss
 from .relational_kd import AngleWiseRKD, DistanceWiseRKD
@@ -21,5 +22,5 @@
     'ChannelWiseDivergence', 'KLDivergence', 'AngleWiseRKD', 'DistanceWiseRKD',
     'WSLD', 'L2Loss', 'ABLoss', 'DKDLoss', 'KDSoftCELoss', 'ActivationLoss',
     'OnehotLikeLoss', 'InformationEntropyLoss', 'FTLoss', 'ATLoss', 'OFDLoss',
-    'L1Loss', 'FBKDLoss', 'CRDLoss', 'CrossEntropyLoss', 'PKDLoss'
+    'L1Loss', 'FBKDLoss', 'CRDLoss', 'CrossEntropyLoss', 'PKDLoss', 'MGDLoss'
 ]
diff --git a/mmrazor/models/losses/mgd_loss.py b/mmrazor/models/losses/mgd_loss.py
new file mode 100644
index 000000000..1321edc53
--- /dev/null
+++ b/mmrazor/models/losses/mgd_loss.py
@@ -0,0 +1,54 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+
+import torch
+import torch.nn as nn
+
+from mmrazor.registry import MODELS
+
+
+@MODELS.register_module()
+class MGDLoss(nn.Module):
+    """PyTorch version of `Masked Generative Distillation.
+
+    <https://arxiv.org/abs/2205.01529>`
+
+    Args:
+        alpha_mgd (float, optional): Weight of dis_loss. Defaults to 0.00002
+    """
+
+    def __init__(self, alpha_mgd: float = 0.00002) -> None:
+        super(MGDLoss, self).__init__()
+        self.alpha_mgd = alpha_mgd
+        self.loss_mse = nn.MSELoss(reduction='sum')
+
+    def forward(self, preds_S: torch.Tensor,
+                preds_T: torch.Tensor) -> torch.Tensor:
+        """Forward function.
+
+        Args:
+            preds_S(torch.Tensor): Bs*C*H*W, student's feature map
+            preds_T(torch.Tensor): Bs*C*H*W, teacher's feature map
+
+        Return:
+            torch.Tensor: The calculated loss value.
+        """
+        assert preds_S.shape == preds_T.shape
+        loss = self.get_dis_loss(preds_S, preds_T) * self.alpha_mgd
+
+        return loss
+
+    def get_dis_loss(self, preds_S: torch.Tensor,
+                     preds_T: torch.Tensor) -> torch.Tensor:
+        """Get MSE distance of preds_S and preds_T.
+
+        Args:
+            preds_S(torch.Tensor): Bs*C*H*W, student's feature map
+            preds_T(torch.Tensor): Bs*C*H*W, teacher's feature map
+
+        Return:
+            torch.Tensor: The calculated mse distance value.
+        """
+        N, C, H, W = preds_T.shape
+        dis_loss = self.loss_mse(preds_S, preds_T) / N
+
+        return dis_loss


Test patch:
diff --git a/tests/test_models/test_architectures/test_connectors/test_connectors.py b/tests/test_models/test_architectures/test_connectors/test_connectors.py
index 5d44efd75..0bb8e1ef7 100644
--- a/tests/test_models/test_architectures/test_connectors/test_connectors.py
+++ b/tests/test_models/test_architectures/test_connectors/test_connectors.py
@@ -5,8 +5,9 @@
 
 from mmrazor.models import (BYOTConnector, ConvModuleConncetor, CRDConnector,
                             FBKDStudentConnector, FBKDTeacherConnector,
-                            Paraphraser, TorchFunctionalConnector,
-                            TorchNNConnector, Translator)
+                            MGDConnector, Paraphraser,
+                            TorchFunctionalConnector, TorchNNConnector,
+                            Translator)
 
 
 class TestConnector(TestCase):
@@ -130,3 +131,15 @@ def test_torch_connector(self):
             functional_pool_connector = TorchFunctionalConnector()
         with self.assertRaises(ValueError):
             functional_pool_connector = TorchNNConnector(module_name='fake')
+
+    def test_mgd_connector(self):
+        s_feat = torch.randn(1, 16, 8, 8)
+        mgd_connector1 = MGDConnector(
+            student_channels=16, teacher_channels=16, lambda_mgd=0.65)
+        mgd_connector2 = MGDConnector(
+            student_channels=16, teacher_channels=32, lambda_mgd=0.65)
+        s_output1 = mgd_connector1.forward_train(s_feat)
+        s_output2 = mgd_connector2.forward_train(s_feat)
+
+        assert s_output1.shape == torch.Size([1, 16, 8, 8])
+        assert s_output2.shape == torch.Size([1, 32, 8, 8])
diff --git a/tests/test_models/test_losses/test_distillation_losses.py b/tests/test_models/test_losses/test_distillation_losses.py
index 15243650c..77233b81f 100644
--- a/tests/test_models/test_losses/test_distillation_losses.py
+++ b/tests/test_models/test_losses/test_distillation_losses.py
@@ -7,7 +7,8 @@
 from mmrazor import digit_version
 from mmrazor.models import (ABLoss, ActivationLoss, ATLoss, CRDLoss, DKDLoss,
                             FBKDLoss, FTLoss, InformationEntropyLoss,
-                            KDSoftCELoss, OFDLoss, OnehotLikeLoss, PKDLoss)
+                            KDSoftCELoss, MGDLoss, OFDLoss, OnehotLikeLoss,
+                            PKDLoss)
 
 
 class TestLosses(TestCase):
@@ -204,3 +205,9 @@ def test_pkdloss(self):
         loss = pkd_loss(feats_S, feats_T)
         self.assertTrue(loss.numel() == 1)
         self.assertTrue(0. <= loss <= 1.)
+
+    def test_mgd_loss(self):
+        mgd_loss = MGDLoss(alpha_mgd=0.00002)
+        feats_S, feats_T = torch.rand(2, 256, 4, 4), torch.rand(2, 256, 4, 4)
+        loss = mgd_loss(feats_S, feats_T)
+        self.assertTrue(loss.numel() == 1)