Repository: open-mmlab/mmaction2 (language: python)

Issue:
Config refactoring
Ref: https://github.com/open-mmlab/mmdetection/pull/4347




Discussion / hints:

Fix patch:
diff --git a/configs/_base_/models/audioonly_r50.py b/configs/_base_/models/audioonly_r50.py
index a2e01153ad..d4a190c819 100644
--- a/configs/_base_/models/audioonly_r50.py
+++ b/configs/_base_/models/audioonly_r50.py
@@ -12,7 +12,7 @@
         num_classes=400,
         in_channels=1024,
         dropout_ratio=0.5,
-        init_std=0.01))
-# model training and testing settings
-train_cfg = None
-test_cfg = dict(average_clips='prob')
+        init_std=0.01),
+    # model training and testing settings
+    train_cfg=None,
+    test_cfg=dict(average_clips='prob'))
diff --git a/configs/_base_/models/c3d_sports1m_pretrained.py b/configs/_base_/models/c3d_sports1m_pretrained.py
index e1a5f45959..1cdc3d49f9 100644
--- a/configs/_base_/models/c3d_sports1m_pretrained.py
+++ b/configs/_base_/models/c3d_sports1m_pretrained.py
@@ -17,8 +17,7 @@
         in_channels=4096,
         spatial_type=None,
         dropout_ratio=0.5,
-        init_std=0.01))
-
-# model training and testing settings
-train_cfg = None
-test_cfg = dict(average_clips='score')
+        init_std=0.01),
+    # model training and testing settings
+    train_cfg=None,
+    test_cfg=dict(average_clips='score'))
diff --git a/configs/_base_/models/csn_ig65m_pretrained.py b/configs/_base_/models/csn_ig65m_pretrained.py
index c44aed4c9a..2e827026d5 100644
--- a/configs/_base_/models/csn_ig65m_pretrained.py
+++ b/configs/_base_/models/csn_ig65m_pretrained.py
@@ -17,7 +17,7 @@
         in_channels=2048,
         spatial_type='avg',
         dropout_ratio=0.5,
-        init_std=0.01))
-# model training and testing settings
-train_cfg = None
-test_cfg = dict(average_clips='prob')
+        init_std=0.01),
+    # model training and testing settings
+    train_cfg=None,
+    test_cfg=dict(average_clips='prob'))
diff --git a/configs/_base_/models/i3d_r50.py b/configs/_base_/models/i3d_r50.py
index acb0654fb5..629ea148a7 100644
--- a/configs/_base_/models/i3d_r50.py
+++ b/configs/_base_/models/i3d_r50.py
@@ -16,7 +16,7 @@
         in_channels=2048,
         spatial_type='avg',
         dropout_ratio=0.5,
-        init_std=0.01))
-# model training and testing settings
-train_cfg = None
-test_cfg = dict(average_clips='prob')
+        init_std=0.01),
+    # model training and testing settings
+    train_cfg=None,
+    test_cfg=dict(average_clips='prob'))
diff --git a/configs/_base_/models/r2plus1d_r34.py b/configs/_base_/models/r2plus1d_r34.py
index 0e1598d13d..b5bcdac0c2 100644
--- a/configs/_base_/models/r2plus1d_r34.py
+++ b/configs/_base_/models/r2plus1d_r34.py
@@ -22,7 +22,7 @@
         in_channels=512,
         spatial_type='avg',
         dropout_ratio=0.5,
-        init_std=0.01))
-# model training and testing settings
-train_cfg = None
-test_cfg = dict(average_clips='prob')
+        init_std=0.01),
+    # model training and testing settings
+    train_cfg=None,
+    test_cfg=dict(average_clips='prob'))
diff --git a/configs/_base_/models/slowfast_r50.py b/configs/_base_/models/slowfast_r50.py
index 6a87952151..afa8aab045 100644
--- a/configs/_base_/models/slowfast_r50.py
+++ b/configs/_base_/models/slowfast_r50.py
@@ -33,6 +33,7 @@
         in_channels=2304,  # 2048+256
         num_classes=400,
         spatial_type='avg',
-        dropout_ratio=0.5))
-train_cfg = None
-test_cfg = dict(average_clips='prob')
+        dropout_ratio=0.5),
+    # model training and testing settings
+    train_cfg=None,
+    test_cfg=dict(average_clips='prob'))
diff --git a/configs/_base_/models/slowonly_r50.py b/configs/_base_/models/slowonly_r50.py
index fa6aa4d130..13081786bf 100644
--- a/configs/_base_/models/slowonly_r50.py
+++ b/configs/_base_/models/slowonly_r50.py
@@ -16,6 +16,7 @@
         in_channels=2048,
         num_classes=400,
         spatial_type='avg',
-        dropout_ratio=0.5))
-train_cfg = None
-test_cfg = dict(average_clips='prob')
+        dropout_ratio=0.5),
+    # model training and testing settings
+    train_cfg=None,
+    test_cfg=dict(average_clips='prob'))
diff --git a/configs/_base_/models/tanet_r50.py b/configs/_base_/models/tanet_r50.py
index e9f3230ace..b20ea82215 100644
--- a/configs/_base_/models/tanet_r50.py
+++ b/configs/_base_/models/tanet_r50.py
@@ -14,6 +14,7 @@
         spatial_type='avg',
         consensus=dict(type='AvgConsensus', dim=1),
         dropout_ratio=0.5,
-        init_std=0.001))
-train_cfg = None
-test_cfg = dict(average_clips='prob')
+        init_std=0.001),
+    # model training and testing settings
+    train_cfg=None,
+    test_cfg=dict(average_clips='prob'))
diff --git a/configs/_base_/models/tin_r50.py b/configs/_base_/models/tin_r50.py
index 4bb5be3209..af9ac373e1 100644
--- a/configs/_base_/models/tin_r50.py
+++ b/configs/_base_/models/tin_r50.py
@@ -15,7 +15,7 @@
         consensus=dict(type='AvgConsensus', dim=1),
         dropout_ratio=0.5,
         init_std=0.001,
-        is_shift=False))
-# model training and testing settings
-train_cfg = None
-test_cfg = dict(average_clips=None)
+        is_shift=False),
+    # model training and testing settings
+    train_cfg=None,
+    test_cfg=dict(average_clips=None))
diff --git a/configs/_base_/models/tpn_slowonly_r50.py b/configs/_base_/models/tpn_slowonly_r50.py
index 0845274c26..072e5e8872 100644
--- a/configs/_base_/models/tpn_slowonly_r50.py
+++ b/configs/_base_/models/tpn_slowonly_r50.py
@@ -34,6 +34,7 @@
         spatial_type='avg',
         consensus=dict(type='AvgConsensus', dim=1),
         dropout_ratio=0.5,
-        init_std=0.01))
-train_cfg = None
-test_cfg = dict(average_clips='prob')
+        init_std=0.01),
+    # model training and testing settings
+    train_cfg=None,
+    test_cfg=dict(average_clips='prob'))
diff --git a/configs/_base_/models/tpn_tsm_r50.py b/configs/_base_/models/tpn_tsm_r50.py
index 657f887fd6..765bd4a2d4 100644
--- a/configs/_base_/models/tpn_tsm_r50.py
+++ b/configs/_base_/models/tpn_tsm_r50.py
@@ -30,6 +30,7 @@
         spatial_type='avg',
         consensus=dict(type='AvgConsensus', dim=1),
         dropout_ratio=0.5,
-        init_std=0.01))
-train_cfg = None
-test_cfg = dict(average_clips=None)
+        init_std=0.01),
+    # model training and testing settings
+    train_cfg=None,
+    test_cfg=dict(average_clips=None))
diff --git a/configs/_base_/models/tsm_mobilenet_v2.py b/configs/_base_/models/tsm_mobilenet_v2.py
index 0338e19e27..bce81074e2 100644
--- a/configs/_base_/models/tsm_mobilenet_v2.py
+++ b/configs/_base_/models/tsm_mobilenet_v2.py
@@ -16,7 +16,7 @@
         consensus=dict(type='AvgConsensus', dim=1),
         dropout_ratio=0.5,
         init_std=0.001,
-        is_shift=True))
-# model training and testing settings
-train_cfg = None
-test_cfg = dict(average_clips='prob')
+        is_shift=True),
+    # model training and testing settings
+    train_cfg=None,
+    test_cfg=dict(average_clips='prob'))
diff --git a/configs/_base_/models/tsm_r50.py b/configs/_base_/models/tsm_r50.py
index 70bc9ef321..477497b679 100644
--- a/configs/_base_/models/tsm_r50.py
+++ b/configs/_base_/models/tsm_r50.py
@@ -15,7 +15,7 @@
         consensus=dict(type='AvgConsensus', dim=1),
         dropout_ratio=0.5,
         init_std=0.001,
-        is_shift=True))
-# model training and testing settings
-train_cfg = None
-test_cfg = dict(average_clips='prob')
+        is_shift=True),
+    # model training and testing settings
+    train_cfg=None,
+    test_cfg=dict(average_clips='prob'))
diff --git a/configs/_base_/models/tsn_r50.py b/configs/_base_/models/tsn_r50.py
index b626e7db7c..d879ea692a 100644
--- a/configs/_base_/models/tsn_r50.py
+++ b/configs/_base_/models/tsn_r50.py
@@ -13,7 +13,7 @@
         spatial_type='avg',
         consensus=dict(type='AvgConsensus', dim=1),
         dropout_ratio=0.4,
-        init_std=0.01))
-# model training and testing settings
-train_cfg = None
-test_cfg = dict(average_clips=None)
+        init_std=0.01),
+    # model training and testing settings
+    train_cfg=None,
+    test_cfg=dict(average_clips=None))
diff --git a/configs/_base_/models/tsn_r50_audio.py b/configs/_base_/models/tsn_r50_audio.py
index 5a2f9ecaf5..2c3ab0dff5 100644
--- a/configs/_base_/models/tsn_r50_audio.py
+++ b/configs/_base_/models/tsn_r50_audio.py
@@ -7,7 +7,7 @@
         num_classes=400,
         in_channels=2048,
         dropout_ratio=0.5,
-        init_std=0.01))
-# model training and testing settings
-train_cfg = None
-test_cfg = dict(average_clips='prob')
+        init_std=0.01),
+    # model training and testing settings
+    train_cfg=None,
+    test_cfg=dict(average_clips='prob'))
diff --git a/configs/_base_/models/x3d.py b/configs/_base_/models/x3d.py
index c6b44e0c31..10e302055a 100644
--- a/configs/_base_/models/x3d.py
+++ b/configs/_base_/models/x3d.py
@@ -8,7 +8,7 @@
         num_classes=400,
         spatial_type='avg',
         dropout_ratio=0.5,
-        fc1_bias=False))
-# model training and testing settings
-train_cfg = None
-test_cfg = dict(average_clips='prob')
+        fc1_bias=False),
+    # model training and testing settings
+    train_cfg=None,
+    test_cfg=dict(average_clips='prob'))
diff --git a/configs/detection/ava/slowfast_context_kinetics_pretrained_r50_4x16x1_20e_ava_rgb.py b/configs/detection/ava/slowfast_context_kinetics_pretrained_r50_4x16x1_20e_ava_rgb.py
index 1366382a6d..ec208c27db 100644
--- a/configs/detection/ava/slowfast_context_kinetics_pretrained_r50_4x16x1_20e_ava_rgb.py
+++ b/configs/detection/ava/slowfast_context_kinetics_pretrained_r50_4x16x1_20e_ava_rgb.py
@@ -41,24 +41,23 @@
             in_channels=4608,
             num_classes=81,
             multilabel=True,
-            dropout_ratio=0.5)))
-
-train_cfg = dict(
-    rcnn=dict(
-        assigner=dict(
-            type='MaxIoUAssignerAVA',
-            pos_iou_thr=0.9,
-            neg_iou_thr=0.9,
-            min_pos_iou=0.9),
-        sampler=dict(
-            type='RandomSampler',
-            num=32,
-            pos_fraction=1,
-            neg_pos_ub=-1,
-            add_gt_as_proposals=True),
-        pos_weight=1.0,
-        debug=False))
-test_cfg = dict(rcnn=dict(action_thr=0.00))
+            dropout_ratio=0.5)),
+    train_cfg=dict(
+        rcnn=dict(
+            assigner=dict(
+                type='MaxIoUAssignerAVA',
+                pos_iou_thr=0.9,
+                neg_iou_thr=0.9,
+                min_pos_iou=0.9),
+            sampler=dict(
+                type='RandomSampler',
+                num=32,
+                pos_fraction=1,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=True),
+            pos_weight=1.0,
+            debug=False)),
+    test_cfg=dict(rcnn=dict(action_thr=0.00)))
 
 dataset_type = 'AVADataset'
 data_root = 'data/ava/rawframes'
diff --git a/configs/detection/ava/slowfast_kinetics_pretrained_r50_4x16x1_20e_ava_rgb.py b/configs/detection/ava/slowfast_kinetics_pretrained_r50_4x16x1_20e_ava_rgb.py
index 47ca925ce9..58a7c40d4d 100644
--- a/configs/detection/ava/slowfast_kinetics_pretrained_r50_4x16x1_20e_ava_rgb.py
+++ b/configs/detection/ava/slowfast_kinetics_pretrained_r50_4x16x1_20e_ava_rgb.py
@@ -40,24 +40,23 @@
             in_channels=2304,
             num_classes=81,
             multilabel=True,
-            dropout_ratio=0.5)))
-
-train_cfg = dict(
-    rcnn=dict(
-        assigner=dict(
-            type='MaxIoUAssignerAVA',
-            pos_iou_thr=0.9,
-            neg_iou_thr=0.9,
-            min_pos_iou=0.9),
-        sampler=dict(
-            type='RandomSampler',
-            num=32,
-            pos_fraction=1,
-            neg_pos_ub=-1,
-            add_gt_as_proposals=True),
-        pos_weight=1.0,
-        debug=False))
-test_cfg = dict(rcnn=dict(action_thr=0.00))
+            dropout_ratio=0.5)),
+    train_cfg=dict(
+        rcnn=dict(
+            assigner=dict(
+                type='MaxIoUAssignerAVA',
+                pos_iou_thr=0.9,
+                neg_iou_thr=0.9,
+                min_pos_iou=0.9),
+            sampler=dict(
+                type='RandomSampler',
+                num=32,
+                pos_fraction=1,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=True),
+            pos_weight=1.0,
+            debug=False)),
+    test_cfg=dict(rcnn=dict(action_thr=0.00)))
 
 dataset_type = 'AVADataset'
 data_root = 'data/ava/rawframes'
diff --git a/configs/detection/ava/slowfast_kinetics_pretrained_r50_8x8x1_20e_ava_rgb.py b/configs/detection/ava/slowfast_kinetics_pretrained_r50_8x8x1_20e_ava_rgb.py
index a81ec62234..7044d3e686 100644
--- a/configs/detection/ava/slowfast_kinetics_pretrained_r50_8x8x1_20e_ava_rgb.py
+++ b/configs/detection/ava/slowfast_kinetics_pretrained_r50_8x8x1_20e_ava_rgb.py
@@ -40,24 +40,23 @@
             in_channels=2304,
             num_classes=81,
             multilabel=True,
-            dropout_ratio=0.5)))
-
-train_cfg = dict(
-    rcnn=dict(
-        assigner=dict(
-            type='MaxIoUAssignerAVA',
-            pos_iou_thr=0.9,
-            neg_iou_thr=0.9,
-            min_pos_iou=0.9),
-        sampler=dict(
-            type='RandomSampler',
-            num=32,
-            pos_fraction=1,
-            neg_pos_ub=-1,
-            add_gt_as_proposals=True),
-        pos_weight=1.0,
-        debug=False))
-test_cfg = dict(rcnn=dict(action_thr=0.00))
+            dropout_ratio=0.5)),
+    train_cfg=dict(
+        rcnn=dict(
+            assigner=dict(
+                type='MaxIoUAssignerAVA',
+                pos_iou_thr=0.9,
+                neg_iou_thr=0.9,
+                min_pos_iou=0.9),
+            sampler=dict(
+                type='RandomSampler',
+                num=32,
+                pos_fraction=1,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=True),
+            pos_weight=1.0,
+            debug=False)),
+    test_cfg=dict(rcnn=dict(action_thr=0.00)))
 
 dataset_type = 'AVADataset'
 data_root = 'data/ava/rawframes'
diff --git a/configs/detection/ava/slowonly_kinetics_pretrained_r101_8x8x1_20e_ava_rgb.py b/configs/detection/ava/slowonly_kinetics_pretrained_r101_8x8x1_20e_ava_rgb.py
index ecdbde0ccc..374bef8a36 100644
--- a/configs/detection/ava/slowonly_kinetics_pretrained_r101_8x8x1_20e_ava_rgb.py
+++ b/configs/detection/ava/slowonly_kinetics_pretrained_r101_8x8x1_20e_ava_rgb.py
@@ -24,24 +24,23 @@
             in_channels=2048,
             num_classes=81,
             multilabel=True,
-            dropout_ratio=0.5)))
-
-train_cfg = dict(
-    rcnn=dict(
-        assigner=dict(
-            type='MaxIoUAssignerAVA',
-            pos_iou_thr=0.9,
-            neg_iou_thr=0.9,
-            min_pos_iou=0.9),
-        sampler=dict(
-            type='RandomSampler',
-            num=32,
-            pos_fraction=1,
-            neg_pos_ub=-1,
-            add_gt_as_proposals=True),
-        pos_weight=1.0,
-        debug=False))
-test_cfg = dict(rcnn=dict(action_thr=0.00))
+            dropout_ratio=0.5)),
+    train_cfg=dict(
+        rcnn=dict(
+            assigner=dict(
+                type='MaxIoUAssignerAVA',
+                pos_iou_thr=0.9,
+                neg_iou_thr=0.9,
+                min_pos_iou=0.9),
+            sampler=dict(
+                type='RandomSampler',
+                num=32,
+                pos_fraction=1,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=True),
+            pos_weight=1.0,
+            debug=False)),
+    test_cfg=dict(rcnn=dict(action_thr=0.00)))
 
 dataset_type = 'AVADataset'
 data_root = 'data/ava/rawframes'
diff --git a/configs/detection/ava/slowonly_kinetics_pretrained_r50_4x16x1_20e_ava_rgb.py b/configs/detection/ava/slowonly_kinetics_pretrained_r50_4x16x1_20e_ava_rgb.py
index e6edb13d7a..3080c5e709 100644
--- a/configs/detection/ava/slowonly_kinetics_pretrained_r50_4x16x1_20e_ava_rgb.py
+++ b/configs/detection/ava/slowonly_kinetics_pretrained_r50_4x16x1_20e_ava_rgb.py
@@ -24,24 +24,23 @@
             in_channels=2048,
             num_classes=81,
             multilabel=True,
-            dropout_ratio=0.5)))
-
-train_cfg = dict(
-    rcnn=dict(
-        assigner=dict(
-            type='MaxIoUAssignerAVA',
-            pos_iou_thr=0.9,
-            neg_iou_thr=0.9,
-            min_pos_iou=0.9),
-        sampler=dict(
-            type='RandomSampler',
-            num=32,
-            pos_fraction=1,
-            neg_pos_ub=-1,
-            add_gt_as_proposals=True),
-        pos_weight=1.0,
-        debug=False))
-test_cfg = dict(rcnn=dict(action_thr=0.00))
+            dropout_ratio=0.5)),
+    train_cfg=dict(
+        rcnn=dict(
+            assigner=dict(
+                type='MaxIoUAssignerAVA',
+                pos_iou_thr=0.9,
+                neg_iou_thr=0.9,
+                min_pos_iou=0.9),
+            sampler=dict(
+                type='RandomSampler',
+                num=32,
+                pos_fraction=1,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=True),
+            pos_weight=1.0,
+            debug=False)),
+    test_cfg=dict(rcnn=dict(action_thr=0.00)))
 
 dataset_type = 'AVADataset'
 data_root = 'data/ava/rawframes'
diff --git a/configs/detection/ava/slowonly_kinetics_pretrained_r50_4x16x1_20e_ava_rgb_custom_classes.py b/configs/detection/ava/slowonly_kinetics_pretrained_r50_4x16x1_20e_ava_rgb_custom_classes.py
index 07b8a3d6c0..e3608b7acb 100644
--- a/configs/detection/ava/slowonly_kinetics_pretrained_r50_4x16x1_20e_ava_rgb_custom_classes.py
+++ b/configs/detection/ava/slowonly_kinetics_pretrained_r50_4x16x1_20e_ava_rgb_custom_classes.py
@@ -31,24 +31,23 @@
             num_classes=num_classes,
             multilabel=True,
             topk=(3, 5),
-            dropout_ratio=0.5)))
-
-train_cfg = dict(
-    rcnn=dict(
-        assigner=dict(
-            type='MaxIoUAssignerAVA',
-            pos_iou_thr=0.9,
-            neg_iou_thr=0.9,
-            min_pos_iou=0.9),
-        sampler=dict(
-            type='RandomSampler',
-            num=32,
-            pos_fraction=1,
-            neg_pos_ub=-1,
-            add_gt_as_proposals=True),
-        pos_weight=1.0,
-        debug=False))
-test_cfg = dict(rcnn=dict(action_thr=0.00))
+            dropout_ratio=0.5)),
+    train_cfg=dict(
+        rcnn=dict(
+            assigner=dict(
+                type='MaxIoUAssignerAVA',
+                pos_iou_thr=0.9,
+                neg_iou_thr=0.9,
+                min_pos_iou=0.9),
+            sampler=dict(
+                type='RandomSampler',
+                num=32,
+                pos_fraction=1,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=True),
+            pos_weight=1.0,
+            debug=False)),
+    test_cfg=dict(rcnn=dict(action_thr=0.00)))
 
 dataset_type = 'AVADataset'
 data_root = 'data/ava/rawframes'
diff --git a/configs/detection/ava/slowonly_omnisource_pretrained_r101_8x8x1_20e_ava_rgb.py b/configs/detection/ava/slowonly_omnisource_pretrained_r101_8x8x1_20e_ava_rgb.py
index 5fa1b7e596..11b3737341 100644
--- a/configs/detection/ava/slowonly_omnisource_pretrained_r101_8x8x1_20e_ava_rgb.py
+++ b/configs/detection/ava/slowonly_omnisource_pretrained_r101_8x8x1_20e_ava_rgb.py
@@ -24,24 +24,23 @@
             in_channels=2048,
             num_classes=81,
             multilabel=True,
-            dropout_ratio=0.5)))
-
-train_cfg = dict(
-    rcnn=dict(
-        assigner=dict(
-            type='MaxIoUAssignerAVA',
-            pos_iou_thr=0.9,
-            neg_iou_thr=0.9,
-            min_pos_iou=0.9),
-        sampler=dict(
-            type='RandomSampler',
-            num=32,
-            pos_fraction=1,
-            neg_pos_ub=-1,
-            add_gt_as_proposals=True),
-        pos_weight=1.0,
-        debug=False))
-test_cfg = dict(rcnn=dict(action_thr=0.00))
+            dropout_ratio=0.5)),
+    train_cfg=dict(
+        rcnn=dict(
+            assigner=dict(
+                type='MaxIoUAssignerAVA',
+                pos_iou_thr=0.9,
+                neg_iou_thr=0.9,
+                min_pos_iou=0.9),
+            sampler=dict(
+                type='RandomSampler',
+                num=32,
+                pos_fraction=1,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=True),
+            pos_weight=1.0,
+            debug=False)),
+    test_cfg=dict(rcnn=dict(action_thr=0.00)))
 
 dataset_type = 'AVADataset'
 data_root = 'data/ava/rawframes'
diff --git a/configs/detection/ava/slowonly_omnisource_pretrained_r50_4x16x1_20e_ava_rgb.py b/configs/detection/ava/slowonly_omnisource_pretrained_r50_4x16x1_20e_ava_rgb.py
index 283417df85..bd320c19fa 100644
--- a/configs/detection/ava/slowonly_omnisource_pretrained_r50_4x16x1_20e_ava_rgb.py
+++ b/configs/detection/ava/slowonly_omnisource_pretrained_r50_4x16x1_20e_ava_rgb.py
@@ -24,24 +24,23 @@
             in_channels=2048,
             num_classes=81,
             multilabel=True,
-            dropout_ratio=0.5)))
-
-train_cfg = dict(
-    rcnn=dict(
-        assigner=dict(
-            type='MaxIoUAssignerAVA',
-            pos_iou_thr=0.9,
-            neg_iou_thr=0.9,
-            min_pos_iou=0.9),
-        sampler=dict(
-            type='RandomSampler',
-            num=32,
-            pos_fraction=1,
-            neg_pos_ub=-1,
-            add_gt_as_proposals=True),
-        pos_weight=1.0,
-        debug=False))
-test_cfg = dict(rcnn=dict(action_thr=0.00))
+            dropout_ratio=0.5)),
+    train_cfg=dict(
+        rcnn=dict(
+            assigner=dict(
+                type='MaxIoUAssignerAVA',
+                pos_iou_thr=0.9,
+                neg_iou_thr=0.9,
+                min_pos_iou=0.9),
+            sampler=dict(
+                type='RandomSampler',
+                num=32,
+                pos_fraction=1,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=True),
+            pos_weight=1.0,
+            debug=False)),
+    test_cfg=dict(rcnn=dict(action_thr=0.00)))
 
 dataset_type = 'AVADataset'
 data_root = 'data/ava/rawframes'
diff --git a/configs/recognition/i3d/i3d_r50_video_imgaug_32x2x1_100e_kinetics400_rgb.py b/configs/recognition/i3d/i3d_r50_video_imgaug_32x2x1_100e_kinetics400_rgb.py
index c6587f6027..dfda8749e8 100644
--- a/configs/recognition/i3d/i3d_r50_video_imgaug_32x2x1_100e_kinetics400_rgb.py
+++ b/configs/recognition/i3d/i3d_r50_video_imgaug_32x2x1_100e_kinetics400_rgb.py
@@ -1,25 +1,5 @@
-# model settings
-model = dict(
-    type='Recognizer3D',
-    backbone=dict(
-        type='ResNet3d',
-        pretrained2d=True,
-        pretrained='torchvision://resnet50',
-        depth=50,
-        conv_cfg=dict(type='Conv3d'),
-        norm_eval=False,
-        inflate=((1, 1, 1), (1, 0, 1, 0), (1, 0, 1, 0, 1, 0), (0, 1, 0)),
-        zero_init_residual=False),
-    cls_head=dict(
-        type='I3DHead',
-        num_classes=400,
-        in_channels=2048,
-        spatial_type='avg',
-        dropout_ratio=0.5,
-        init_std=0.01))
-# model training and testing settings
-train_cfg = None
-test_cfg = dict(average_clips='prob')
+_base_ = ['../../_base_/models/i3d_r50.py']
+
 # dataset settings
 dataset_type = 'VideoDataset'
 data_root = 'data/kinetics400/videos_train'
diff --git a/configs/recognition/r2plus1d/r2plus1d_r34_video_inference_8x8x1_180e_kinetics400_rgb.py b/configs/recognition/r2plus1d/r2plus1d_r34_video_inference_8x8x1_180e_kinetics400_rgb.py
index 66df5184fe..cb4bb161d4 100644
--- a/configs/recognition/r2plus1d/r2plus1d_r34_video_inference_8x8x1_180e_kinetics400_rgb.py
+++ b/configs/recognition/r2plus1d/r2plus1d_r34_video_inference_8x8x1_180e_kinetics400_rgb.py
@@ -3,8 +3,6 @@
 # model settings
 model = dict(backbone=dict(act_cfg=dict(type='ReLU')))
 
-# model training and testing settings
-test_cfg = dict(average_clips='prob')
 # dataset settings
 dataset_type = 'VideoDataset'
 img_norm_cfg = dict(
diff --git a/configs/recognition/slowonly/data_benchmark/slowonly_r50_randomresizedcrop_256p_4x16x1_256e_kinetics400_rgb.py b/configs/recognition/slowonly/data_benchmark/slowonly_r50_randomresizedcrop_256p_4x16x1_256e_kinetics400_rgb.py
index f4564bdcd8..5fbea21781 100644
--- a/configs/recognition/slowonly/data_benchmark/slowonly_r50_randomresizedcrop_256p_4x16x1_256e_kinetics400_rgb.py
+++ b/configs/recognition/slowonly/data_benchmark/slowonly_r50_randomresizedcrop_256p_4x16x1_256e_kinetics400_rgb.py
@@ -15,9 +15,10 @@
         in_channels=2048,
         num_classes=400,
         spatial_type='avg',
-        dropout_ratio=0.5))
-train_cfg = None
-test_cfg = dict(average_clips='prob')
+        dropout_ratio=0.5),
+    train_cfg=None,
+    test_cfg=dict(average_clips='prob'))
+
 dataset_type = 'RawframeDataset'
 data_root = 'data/kinetics400/rawframes_train_256p'
 data_root_val = 'data/kinetics400/rawframes_val_256p'
diff --git a/configs/recognition/slowonly/data_benchmark/slowonly_r50_randomresizedcrop_320p_4x16x1_256e_kinetics400_rgb.py b/configs/recognition/slowonly/data_benchmark/slowonly_r50_randomresizedcrop_320p_4x16x1_256e_kinetics400_rgb.py
index 24b12dcdd2..521357178e 100644
--- a/configs/recognition/slowonly/data_benchmark/slowonly_r50_randomresizedcrop_320p_4x16x1_256e_kinetics400_rgb.py
+++ b/configs/recognition/slowonly/data_benchmark/slowonly_r50_randomresizedcrop_320p_4x16x1_256e_kinetics400_rgb.py
@@ -15,9 +15,9 @@
         in_channels=2048,
         num_classes=400,
         spatial_type='avg',
-        dropout_ratio=0.5))
-train_cfg = None
-test_cfg = dict(average_clips='prob')
+        dropout_ratio=0.5),
+    train_cfg=None,
+    test_cfg=dict(average_clips='prob'))
 dataset_type = 'RawframeDataset'
 data_root = 'data/kinetics400/rawframes_train_320p'
 data_root_val = 'data/kinetics400/rawframes_val_320p'
diff --git a/configs/recognition/slowonly/data_benchmark/slowonly_r50_randomresizedcrop_340x256_4x16x1_256e_kinetics400_rgb.py b/configs/recognition/slowonly/data_benchmark/slowonly_r50_randomresizedcrop_340x256_4x16x1_256e_kinetics400_rgb.py
index 97033fa884..d326e9cccb 100644
--- a/configs/recognition/slowonly/data_benchmark/slowonly_r50_randomresizedcrop_340x256_4x16x1_256e_kinetics400_rgb.py
+++ b/configs/recognition/slowonly/data_benchmark/slowonly_r50_randomresizedcrop_340x256_4x16x1_256e_kinetics400_rgb.py
@@ -15,9 +15,9 @@
         in_channels=2048,
         num_classes=400,
         spatial_type='avg',
-        dropout_ratio=0.5))
-train_cfg = None
-test_cfg = dict(average_clips='prob')
+        dropout_ratio=0.5),
+    train_cfg=None,
+    test_cfg=dict(average_clips='prob'))
 dataset_type = 'RawframeDataset'
 data_root = 'data/kinetics400/rawframes_train'
 data_root_val = 'data/kinetics400/rawframes_val'
diff --git a/configs/recognition/tsm/README.md b/configs/recognition/tsm/README.md
index b488874c5c..219c6c89d4 100644
--- a/configs/recognition/tsm/README.md
+++ b/configs/recognition/tsm/README.md
@@ -73,8 +73,6 @@ Notes:
 
 ```python
 ...
-test_cfg = dict(average_clips='prob')
-...
 test_pipeline = [
     dict(
         type='SampleFrames',
diff --git a/configs/recognition/tsn/tsn_r50_video_inference_1x1x3_100e_kinetics400_rgb.py b/configs/recognition/tsn/tsn_r50_video_inference_1x1x3_100e_kinetics400_rgb.py
index 60e311f02c..99dd110ce6 100644
--- a/configs/recognition/tsn/tsn_r50_video_inference_1x1x3_100e_kinetics400_rgb.py
+++ b/configs/recognition/tsn/tsn_r50_video_inference_1x1x3_100e_kinetics400_rgb.py
@@ -1,6 +1,5 @@
 _base_ = ['../../_base_/models/tsn_r50.py']
 
-test_cfg = dict(average_clips='prob')
 # dataset settings
 dataset_type = 'VideoDataset'
 img_norm_cfg = dict(
diff --git a/configs/recognition_audio/resnet/tsn_r18_64x1x1_100e_kinetics400_audio_feature.py b/configs/recognition_audio/resnet/tsn_r18_64x1x1_100e_kinetics400_audio_feature.py
index 3b4b901bf7..e2fbd7c777 100644
--- a/configs/recognition_audio/resnet/tsn_r18_64x1x1_100e_kinetics400_audio_feature.py
+++ b/configs/recognition_audio/resnet/tsn_r18_64x1x1_100e_kinetics400_audio_feature.py
@@ -9,10 +9,10 @@
         num_classes=400,
         in_channels=512,
         dropout_ratio=0.5,
-        init_std=0.01))
-# model training and testing settings
-train_cfg = None
-test_cfg = dict(average_clips='prob')
+        init_std=0.01),
+    # model training and testing settings
+    train_cfg=None,
+    test_cfg=dict(average_clips='prob'))
 # dataset settings
 dataset_type = 'AudioFeatureDataset'
 data_root = 'data/kinetics400/audio_feature_train'
diff --git a/demo/README.md b/demo/README.md
index 70ebb24417..cb6ba108c3 100644
--- a/demo/README.md
+++ b/demo/README.md
@@ -10,7 +10,7 @@
 
 ## Video demo
 
-We provide a demo script to predict the recognition result using a single video. In order to get predict results in range `[0, 1]`, make sure to set `test_cfg = dict(average_clips='prob')` in config file.
+We provide a demo script to predict the recognition result using a single video. In order to get predict results in range `[0, 1]`, make sure to set `model['test_cfg'] = dict(average_clips='prob')` in config file.
 
 ```shell
 python demo/demo.py ${CONFIG_FILE} ${CHECKPOINT_FILE} ${VIDEO_FILE} {LABEL_FILE} [--use-frames] \
@@ -212,7 +212,7 @@ or use checkpoint url from `configs/` to directly load corresponding checkpoint,
 
 ## Webcam demo
 
-We provide a demo script to implement real-time action recognition from web camera. In order to get predict results in range `[0, 1]`, make sure to set `test_cfg = dict(average_clips='prob')` in config file.
+We provide a demo script to implement real-time action recognition from web camera. In order to get predict results in range `[0, 1]`, make sure to set `model.['test_cfg'] = dict(average_clips='prob')` in config file.
 
 ```shell
 python demo/webcam_demo.py ${CONFIG_FILE} ${CHECKPOINT_FILE} ${LABEL_FILE} \
diff --git a/demo/demo_spatiotemporal_det.py b/demo/demo_spatiotemporal_det.py
index 31ecd39ca0..d8362021a5 100644
--- a/demo/demo_spatiotemporal_det.py
+++ b/demo/demo_spatiotemporal_det.py
@@ -319,7 +319,7 @@ def main():
 
     # Build STDET model
     config.model.backbone.pretrained = None
-    model = build_detector(config.model, test_cfg=config.test_cfg)
+    model = build_detector(config.model, test_cfg=config.get('test_cfg'))
 
     load_checkpoint(model, args.checkpoint, map_location=args.device)
     model.to(args.device)
diff --git a/demo/faster_rcnn_r50_fpn_2x_coco.py b/demo/faster_rcnn_r50_fpn_2x_coco.py
index 336086282a..326258e935 100644
--- a/demo/faster_rcnn_r50_fpn_2x_coco.py
+++ b/demo/faster_rcnn_r50_fpn_2x_coco.py
@@ -52,61 +52,62 @@
             reg_class_agnostic=False,
             loss_cls=dict(
                 type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0),
-            loss_bbox=dict(type='L1Loss', loss_weight=1.0))))
-# model training and testing settings
-train_cfg = dict(
-    rpn=dict(
-        assigner=dict(
-            type='MaxIoUAssigner',
-            pos_iou_thr=0.7,
-            neg_iou_thr=0.3,
-            min_pos_iou=0.3,
-            match_low_quality=True,
-            ignore_iof_thr=-1),
-        sampler=dict(
-            type='RandomSampler',
-            num=256,
-            pos_fraction=0.5,
-            neg_pos_ub=-1,
-            add_gt_as_proposals=False),
-        allowed_border=-1,
-        pos_weight=-1,
-        debug=False),
-    rpn_proposal=dict(
-        nms_across_levels=False,
-        nms_pre=2000,
-        nms_post=1000,
-        max_num=1000,
-        nms_thr=0.7,
-        min_bbox_size=0),
-    rcnn=dict(
-        assigner=dict(
-            type='MaxIoUAssigner',
-            pos_iou_thr=0.5,
-            neg_iou_thr=0.5,
-            min_pos_iou=0.5,
-            match_low_quality=False,
-            ignore_iof_thr=-1),
-        sampler=dict(
-            type='RandomSampler',
-            num=512,
-            pos_fraction=0.25,
-            neg_pos_ub=-1,
-            add_gt_as_proposals=True),
-        pos_weight=-1,
-        debug=False))
-test_cfg = dict(
-    rpn=dict(
-        nms_across_levels=False,
-        nms_pre=1000,
-        nms_post=1000,
-        max_num=1000,
-        nms_thr=0.7,
-        min_bbox_size=0),
-    rcnn=dict(
-        score_thr=0.05,
-        nms=dict(type='nms', iou_threshold=0.5),
-        max_per_img=100))
+            loss_bbox=dict(type='L1Loss', loss_weight=1.0))),
+    # model training and testing settings
+    train_cfg=dict(
+        rpn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.3,
+                min_pos_iou=0.3,
+                match_low_quality=True,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=256,
+                pos_fraction=0.5,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=False),
+            allowed_border=-1,
+            pos_weight=-1,
+            debug=False),
+        rpn_proposal=dict(
+            nms_across_levels=False,
+            nms_pre=2000,
+            nms_post=1000,
+            max_num=1000,
+            nms_thr=0.7,
+            min_bbox_size=0),
+        rcnn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.5,
+                neg_iou_thr=0.5,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=512,
+                pos_fraction=0.25,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=True),
+            pos_weight=-1,
+            debug=False)),
+    test_cfg=dict(
+        rpn=dict(
+            nms_across_levels=False,
+            nms_pre=1000,
+            nms_post=1000,
+            max_num=1000,
+            nms_thr=0.7,
+            min_bbox_size=0),
+        rcnn=dict(
+            score_thr=0.05,
+            nms=dict(type='nms', iou_threshold=0.5),
+            max_per_img=100)))
+
 # dataset config
 dataset_type = 'CocoDataset'
 data_root = 'data/coco/'
diff --git a/demo/mmaction2_tutorial.ipynb b/demo/mmaction2_tutorial.ipynb
index 166db999da..82ee6cdb97 100644
--- a/demo/mmaction2_tutorial.ipynb
+++ b/demo/mmaction2_tutorial.ipynb
@@ -136,10 +136,10 @@
       "      Successfully uninstalled mmcv-full-1.2.0\n",
       "Successfully installed mmcv-full-1.2.0\n",
       "Cloning into 'mmaction2'...\n",
-      "remote: Enumerating objects: 8, done.\u001B[K\n",
-      "remote: Counting objects: 100% (8/8), done.\u001B[K\n",
-      "remote: Compressing objects: 100% (8/8), done.\u001B[K\n",
-      "remote: Total 7596 (delta 0), reused 1 (delta 0), pack-reused 7588\u001B[K\n",
+      "remote: Enumerating objects: 8, done.\u001b[K\n",
+      "remote: Counting objects: 100% (8/8), done.\u001b[K\n",
+      "remote: Compressing objects: 100% (8/8), done.\u001b[K\n",
+      "remote: Total 7596 (delta 0), reused 1 (delta 0), pack-reused 7588\u001b[K\n",
       "Receiving objects: 100% (7596/7596), 33.53 MiB | 37.44 MiB/s, done.\n",
       "Resolving deltas: 100% (5329/5329), done.\n",
       "/content/mmaction2\n",
@@ -850,7 +850,7 @@
     "datasets = [build_dataset(cfg.data.train)]\n",
     "\n",
     "# Build the recognizer\n",
-    "model = build_model(cfg.model, train_cfg=cfg.train_cfg, test_cfg=cfg.test_cfg)\n",
+    "model = build_model(cfg.model, train_cfg=cfg.get('train_cfg'), test_cfg=cfg.get('test_cfg'))\n",
     "\n",
     "# Create work_dir\n",
     "mmcv.mkdir_or_exist(osp.abspath(cfg.work_dir))\n",
diff --git a/docs/changelog.md b/docs/changelog.md
index c22efc88f0..ee928ed5e7 100644
--- a/docs/changelog.md
+++ b/docs/changelog.md
@@ -19,6 +19,7 @@
 
 - Training custom classes of ava dataset ([#555](https://github.com/open-mmlab/mmaction2/pull/555))
 - Support soft label for CrossEntropyLoss ([#625](https://github.com/open-mmlab/mmaction2/pull/625))
+- Refactor config: Specify `train_cfg` and `test_cfg` in `model` ([#629](https://github.com/open-mmlab/mmaction2/pull/629))
 
 **Bug and Typo Fixes**
 
diff --git a/docs/faq.md b/docs/faq.md
index f059e8ccb8..e19d55941e 100644
--- a/docs/faq.md
+++ b/docs/faq.md
@@ -67,11 +67,11 @@ If the contents here do not cover your issue, please create an issue using the [
 
 - **How to make predicted score normalized by softmax within [0, 1] ?**
 
-    change this in the config, make `test_cfg = dict(average_clips='prob')`.
+    change this in the config, make `model['test_cfg'] = dict(average_clips='prob')`.
 
 - **What if the model is too large and the GPU memory can not fit even only one testing sample ?**
 
-    By default, the 3d models are tested with 10clips x 3crops, which are 30 views in total. For extremely large models, the GPU memory can not fit even only one testing sample (cuz there are 30 views). To handle this, you can set `max_testing_views=n` in test_cfg of the config file. If so, n views will be used as a batch during forwarding to save GPU memory used.
+    By default, the 3d models are tested with 10clips x 3crops, which are 30 views in total. For extremely large models, the GPU memory can not fit even only one testing sample (cuz there are 30 views). To handle this, you can set `max_testing_views=n` in `model['test_cfg']` of the config file. If so, n views will be used as a batch during forwarding to save GPU memory used.
 
 - **How to show test results ?**
 
diff --git a/docs/tutorials/1_config.md b/docs/tutorials/1_config.md
index 37a1a428bd..c0bcf5d439 100644
--- a/docs/tutorials/1_config.md
+++ b/docs/tutorials/1_config.md
@@ -249,10 +249,10 @@ which is convenient to conduct various experiments.
             spatial_type='avg',  # Type of pooling in spatial dimension
             consensus=dict(type='AvgConsensus', dim=1),  # Config of consensus module
             dropout_ratio=0.4,  # Probability in dropout layer
-            init_std=0.01))  # Std value for linear layer initiation
-    # model training and testing settings
-    train_cfg = None  # Config of training hyperparameters for TSN
-    test_cfg = dict(average_clips=None) # Config for testing hyperparameters for TSN. Here we define clip averaging method in it
+            init_std=0.01), # Std value for linear layer initiation
+            # model training and testing settings
+            train_cfg=None,  # Config of training hyperparameters for TSN
+            test_cfg=dict(average_clips=None))  # Config for testing hyperparameters for TSN.
 
     # dataset settings
     dataset_type = 'RawframeDataset'  # Type of dataset for training, valiation and testing
@@ -444,7 +444,7 @@ We incorporate modular design into our config system, which is convenient to con
 
     ```python
     # model setting
-    model = dict( # Config of the model
+    model = dict(  # Config of the model
         type='FastRCNN',  # Type of the detector
         backbone=dict(  # Dict for backbone
             type='ResNet3dSlowOnly',  # Name of the backbone
@@ -469,27 +469,26 @@ We incorporate modular design into our config system, which is convenient to con
                 in_channels=2048, # Number of channels of the input feature
                 num_classes=81, # Number of action classes + 1
                 multilabel=True,  # If the dataset is multilabel
-                dropout_ratio=0.5)))  # The dropout ratio used
-
-    # model training and testing settings
-    train_cfg = dict( # Training config of FastRCNN
-        rcnn=dict(  # Dict for rcnn training config
-            assigner=dict(  # Dict for assigner
-                type='MaxIoUAssignerAVA', # Name of the assigner
-                pos_iou_thr=0.9,  # IoU threshold for positive examples, > pos_iou_thr -> positive
-                neg_iou_thr=0.9,  # IoU threshold for negative examples, < neg_iou_thr -> negative
-                min_pos_iou=0.9), # Minimum acceptable IoU for positive examples
-            sampler=dict( # Dict for sample
-                type='RandomSampler', # Name of the sampler
-                num=32, # Batch Size of the sampler
-                pos_fraction=1, # Positive bbox fraction of the sampler
-                neg_pos_ub=-1,  # Upper bound of the ratio of num negative to num positive
-                add_gt_as_proposals=True), # Add gt bboxes as proposals
-            pos_weight=1.0, # Loss weight of positive examples
-            debug=False)) # Debug mode
-    test_cfg = dict( # Testing config of FastRCNN
-        rcnn=dict(  # Dict for rcnn testing config
-            action_thr=0.00)) # The threshold of an action
+                dropout_ratio=0.5)),  # The dropout ratio used
+        # model training and testing settings
+        train_cfg=dict(  # Training config of FastRCNN
+            rcnn=dict(  # Dict for rcnn training config
+                assigner=dict(  # Dict for assigner
+                    type='MaxIoUAssignerAVA', # Name of the assigner
+                    pos_iou_thr=0.9,  # IoU threshold for positive examples, > pos_iou_thr -> positive
+                    neg_iou_thr=0.9,  # IoU threshold for negative examples, < neg_iou_thr -> negative
+                    min_pos_iou=0.9), # Minimum acceptable IoU for positive examples
+                sampler=dict( # Dict for sample
+                    type='RandomSampler', # Name of the sampler
+                    num=32, # Batch Size of the sampler
+                    pos_fraction=1, # Positive bbox fraction of the sampler
+                    neg_pos_ub=-1,  # Upper bound of the ratio of num negative to num positive
+                    add_gt_as_proposals=True), # Add gt bboxes as proposals
+                pos_weight=1.0, # Loss weight of positive examples
+                debug=False)), # Debug mode
+        test_cfg=dict( # Testing config of FastRCNN
+            rcnn=dict(  # Dict for rcnn testing config
+                action_thr=0.00))) # The threshold of an action
 
     # dataset settings
     dataset_type = 'AVADataset' # Type of dataset for training, valiation and testing
diff --git a/docs/tutorials/2_finetune.md b/docs/tutorials/2_finetune.md
index 12b819bb0f..206b91b4a4 100644
--- a/docs/tutorials/2_finetune.md
+++ b/docs/tutorials/2_finetune.md
@@ -46,7 +46,9 @@ model = dict(
         spatial_type='avg',
         consensus=dict(type='AvgConsensus', dim=1),
         dropout_ratio=0.4,
-        init_std=0.01))
+        init_std=0.01),
+    train_cfg=None,
+    test_cfg=dict(average_clips=None))
 ```
 
 Note that the `pretrained='torchvision://resnet50'` setting is used for initializing backbone.
diff --git a/mmaction/apis/inference.py b/mmaction/apis/inference.py
index 3af448b4de..008e4e3d31 100644
--- a/mmaction/apis/inference.py
+++ b/mmaction/apis/inference.py
@@ -42,9 +42,11 @@ def init_recognizer(config,
                            f'dataset type in config, but got input type '
                            f"'{input_type}' and dataset type "
                            f"'{config.dataset_type}'")
+
     # pretrained model is unnecessary since we directly load checkpoint later
     config.model.backbone.pretrained = None
-    model = build_recognizer(config.model, test_cfg=config.test_cfg)
+    model = build_recognizer(config.model, test_cfg=config.get('test_cfg'))
+
     if checkpoint is not None:
         load_checkpoint(model, checkpoint, map_location=device)
     model.cfg = config
diff --git a/mmaction/models/builder.py b/mmaction/models/builder.py
index 634c462bbe..7fed5524c2 100644
--- a/mmaction/models/builder.py
+++ b/mmaction/models/builder.py
@@ -1,3 +1,5 @@
+import warnings
+
 import torch.nn as nn
 from mmcv.utils import Registry, build_from_cfg
 
@@ -50,6 +52,16 @@ def build_head(cfg):
 
 def build_recognizer(cfg, train_cfg=None, test_cfg=None):
     """Build recognizer."""
+    if train_cfg is not None or test_cfg is not None:
+        warnings.warn(
+            'train_cfg and test_cfg is deprecated, '
+            'please specify them in model. Details see this '
+            'PR: https://github.com/open-mmlab/mmaction2/pull/629',
+            UserWarning)
+    assert cfg.get('train_cfg') is None or train_cfg is None, \
+        'train_cfg specified in both outer field and model field '
+    assert cfg.get('test_cfg') is None or test_cfg is None, \
+        'test_cfg specified in both outer field and model field '
     return build(cfg, RECOGNIZERS,
                  dict(train_cfg=train_cfg, test_cfg=test_cfg))
 
@@ -73,6 +85,12 @@ def build_model(cfg, train_cfg=None, test_cfg=None):
     if obj_type in RECOGNIZERS:
         return build_recognizer(cfg, train_cfg, test_cfg)
     if obj_type in DETECTORS:
+        if train_cfg is not None or test_cfg is not None:
+            warnings.warn(
+                'train_cfg and test_cfg is deprecated, '
+                'please specify them in model. Details see this '
+                'PR: https://github.com/open-mmlab/mmaction2/pull/629',
+                UserWarning)
         return build_detector(cfg, train_cfg, test_cfg)
     model_in_mmdet = ['FastRCNN']
     if obj_type in model_in_mmdet:
diff --git a/mmaction/models/localizers/bmn.py b/mmaction/models/localizers/bmn.py
index 20efbc5b84..f8a63c9d12 100644
--- a/mmaction/models/localizers/bmn.py
+++ b/mmaction/models/localizers/bmn.py
@@ -291,8 +291,10 @@ def generate_labels(self, gt_bbox):
         for every_gt_bbox in gt_bbox:
             gt_iou_map = []
             for start, end in every_gt_bbox:
-                start = start.numpy()
-                end = end.numpy()
+                if isinstance(start, torch.Tensor):
+                    start = start.numpy()
+                if isinstance(end, torch.Tensor):
+                    end = end.numpy()
                 current_gt_iou_map = temporal_iou(self.match_map[:, 0],
                                                   self.match_map[:, 1], start,
                                                   end)
diff --git a/tools/analysis/benchmark.py b/tools/analysis/benchmark.py
index 298133550e..7965158d67 100644
--- a/tools/analysis/benchmark.py
+++ b/tools/analysis/benchmark.py
@@ -46,7 +46,8 @@ def main():
         shuffle=False)
 
     # build the model and load checkpoint
-    model = build_model(cfg.model, train_cfg=None, test_cfg=cfg.test_cfg)
+    model = build_model(
+        cfg.model, train_cfg=None, test_cfg=cfg.get('test_cfg'))
     fp16_cfg = cfg.get('fp16', None)
     if fp16_cfg is not None:
         wrap_fp16_model(model)
diff --git a/tools/analysis/get_flops.py b/tools/analysis/get_flops.py
index b9cddf3b18..12907077ad 100644
--- a/tools/analysis/get_flops.py
+++ b/tools/analysis/get_flops.py
@@ -45,7 +45,10 @@ def main():
 
     cfg = Config.fromfile(args.config)
     model = build_recognizer(
-        cfg.model, train_cfg=cfg.train_cfg, test_cfg=cfg.test_cfg)
+        cfg.model,
+        train_cfg=cfg.get('train_cfg'),
+        test_cfg=cfg.get('test_cfg'))
+
     model = model.cuda()
     model.eval()
 
diff --git a/tools/data/activitynet/tsn_feature_extraction.py b/tools/data/activitynet/tsn_feature_extraction.py
index d47e7df6ad..858992877f 100644
--- a/tools/data/activitynet/tsn_feature_extraction.py
+++ b/tools/data/activitynet/tsn_feature_extraction.py
@@ -71,7 +71,7 @@ def main():
     data_pipeline = Compose(data_pipeline)
 
     # define TSN R50 model, the model is used as the feature extractor
-    model = dict(
+    model_cfg = dict(
         type='Recognizer2D',
         backbone=dict(
             type='ResNet',
@@ -83,8 +83,9 @@ def main():
             num_classes=200,
             in_channels=2048,
             spatial_type='avg',
-            consensus=dict(type='AvgConsensus', dim=1)))
-    model = build_model(model, test_cfg=dict(average_clips=None))
+            consensus=dict(type='AvgConsensus', dim=1)),
+        test_cfg=dict(average_clips=None))
+    model = build_model(model_cfg)
     # load pretrained weight into the feature extractor
     state_dict = torch.load(args.ckpt)['state_dict']
     model.load_state_dict(state_dict)
diff --git a/tools/pytorch2onnx.py b/tools/pytorch2onnx.py
index 6660b94d96..1358e069b3 100644
--- a/tools/pytorch2onnx.py
+++ b/tools/pytorch2onnx.py
@@ -138,7 +138,8 @@ def parse_args():
         cfg.model.backbone.pretrained = None
 
     # build the model
-    model = build_model(cfg.model, train_cfg=None, test_cfg=cfg.test_cfg)
+    model = build_model(
+        cfg.model, train_cfg=None, test_cfg=cfg.get('test_cfg'))
     model = _convert_batchnorm(model)
 
     # onnx.export does not support kwargs
diff --git a/tools/train.py b/tools/train.py
index 06ea801a1c..14b2de32ad 100644
--- a/tools/train.py
+++ b/tools/train.py
@@ -141,7 +141,9 @@ def main():
     meta['work_dir'] = osp.basename(cfg.work_dir.rstrip('/\\'))
 
     model = build_model(
-        cfg.model, train_cfg=cfg.train_cfg, test_cfg=cfg.test_cfg)
+        cfg.model,
+        train_cfg=cfg.get('train_cfg'),
+        test_cfg=cfg.get('test_cfg'))
 
     register_module_hooks(model.backbone, cfg.module_hooks)
 


Test patch:
diff --git a/tests/test_models/base.py b/tests/test_models/base.py
index d278817d10..898749c00c 100644
--- a/tests/test_models/base.py
+++ b/tests/test_models/base.py
@@ -102,15 +102,12 @@ def get_cfg(config_type, fname):
 
 
 def get_recognizer_cfg(fname):
-    config = get_cfg('recognition', fname)
-    return config.model, config.train_cfg, config.test_cfg
+    return get_cfg('recognition', fname)
 
 
 def get_audio_recognizer_cfg(fname):
-    config = get_cfg('recognition_audio', fname)
-    return config.model, config.train_cfg, config.test_cfg
+    return get_cfg('recognition_audio', fname)
 
 
 def get_localizer_cfg(fname):
-    config = get_cfg('localization', fname)
-    return config.model, config.train_cfg, config.test_cfg
+    return get_cfg('localization', fname)
diff --git a/tests/test_models/test_gradcam.py b/tests/test_models/test_gradcam.py
index fdab84446c..04267db03f 100644
--- a/tests/test_models/test_gradcam.py
+++ b/tests/test_models/test_gradcam.py
@@ -3,7 +3,7 @@
 
 from mmaction.models import build_recognizer
 from mmaction.utils.gradcam_utils import GradCAM
-from .base import generate_gradcam_inputs, get_cfg
+from .base import generate_gradcam_inputs, get_recognizer_cfg
 
 
 def _get_target_shapes(input_shape, num_classes=400, model_type='2D'):
@@ -87,10 +87,9 @@ def _do_test_3D_models(recognizer,
 
 
 def test_tsn():
-    config = get_cfg('recognition',
-                     'tsn/tsn_r50_1x1x3_100e_kinetics400_rgb.py')
+    config = get_recognizer_cfg('tsn/tsn_r50_1x1x3_100e_kinetics400_rgb.py')
     config.model['backbone']['pretrained'] = None
-    recognizer = build_recognizer(config.model, test_cfg=config.test_cfg)
+    recognizer = build_recognizer(config.model)
     recognizer.cfg = config
 
     input_shape = (1, 25, 3, 32, 32)
@@ -100,12 +99,11 @@ def test_tsn():
 
 
 def test_i3d():
-    config = get_cfg('recognition',
-                     'i3d/i3d_r50_32x2x1_100e_kinetics400_rgb.py')
+    config = get_recognizer_cfg('i3d/i3d_r50_32x2x1_100e_kinetics400_rgb.py')
     config.model['backbone']['pretrained2d'] = False
     config.model['backbone']['pretrained'] = None
 
-    recognizer = build_recognizer(config.model, test_cfg=config.test_cfg)
+    recognizer = build_recognizer(config.model)
     recognizer.cfg = config
 
     input_shape = [1, 1, 3, 32, 32, 32]
@@ -115,13 +113,13 @@ def test_i3d():
 
 
 def test_r2plus1d():
-    config = get_cfg('recognition',
-                     'r2plus1d/r2plus1d_r34_8x8x1_180e_kinetics400_rgb.py')
+    config = get_recognizer_cfg(
+        'r2plus1d/r2plus1d_r34_8x8x1_180e_kinetics400_rgb.py')
     config.model['backbone']['pretrained2d'] = False
     config.model['backbone']['pretrained'] = None
     config.model['backbone']['norm_cfg'] = dict(type='BN3d')
 
-    recognizer = build_recognizer(config.model, test_cfg=config.test_cfg)
+    recognizer = build_recognizer(config.model)
     recognizer.cfg = config
 
     input_shape = (1, 3, 3, 8, 32, 32)
@@ -131,10 +129,10 @@ def test_r2plus1d():
 
 
 def test_slowfast():
-    config = get_cfg('recognition',
-                     'slowfast/slowfast_r50_4x16x1_256e_kinetics400_rgb.py')
+    config = get_recognizer_cfg(
+        'slowfast/slowfast_r50_4x16x1_256e_kinetics400_rgb.py')
 
-    recognizer = build_recognizer(config.model, test_cfg=config.test_cfg)
+    recognizer = build_recognizer(config.model)
     recognizer.cfg = config
 
     input_shape = (1, 1, 3, 32, 32, 32)
@@ -144,32 +142,31 @@ def test_slowfast():
 
 
 def test_tsm():
-    config = get_cfg('recognition', 'tsm/tsm_r50_1x1x8_50e_kinetics400_rgb.py')
+    config = get_recognizer_cfg('tsm/tsm_r50_1x1x8_50e_kinetics400_rgb.py')
     config.model['backbone']['pretrained'] = None
     target_layer_name = 'backbone/layer4/1/relu'
 
     # base config
-    recognizer = build_recognizer(config.model, test_cfg=config.test_cfg)
+    recognizer = build_recognizer(config.model)
     recognizer.cfg = config
     input_shape = (1, 8, 3, 32, 32)
     _do_test_2D_models(recognizer, target_layer_name, input_shape)
 
     # test twice sample + 3 crops, 2*3*8=48
-    test_cfg = dict(average_clips='prob')
-    recognizer = build_recognizer(config.model, test_cfg=test_cfg)
+    config.model.test_cfg = dict(average_clips='prob')
+    recognizer = build_recognizer(config.model)
     recognizer.cfg = config
     input_shape = (1, 48, 3, 32, 32)
     _do_test_2D_models(recognizer, target_layer_name, input_shape)
 
 
 def test_csn():
-    config = get_cfg(
-        'recognition',
+    config = get_recognizer_cfg(
         'csn/ircsn_ig65m_pretrained_r152_32x2x1_58e_kinetics400_rgb.py')
     config.model['backbone']['pretrained2d'] = False
     config.model['backbone']['pretrained'] = None
 
-    recognizer = build_recognizer(config.model, test_cfg=config.test_cfg)
+    recognizer = build_recognizer(config.model)
     recognizer.cfg = config
     input_shape = (1, 1, 3, 32, 32, 32)
     target_layer_name = 'backbone/layer4/1/relu'
@@ -180,28 +177,27 @@ def test_csn():
 def test_tpn():
     target_layer_name = 'backbone/layer4/1/relu'
 
-    config = get_cfg('recognition', 'tpn/tpn_tsm_r50_1x1x8_150e_sthv1_rgb.py')
+    config = get_recognizer_cfg('tpn/tpn_tsm_r50_1x1x8_150e_sthv1_rgb.py')
     config.model['backbone']['pretrained'] = None
-    recognizer = build_recognizer(config.model, test_cfg=config.test_cfg)
+    recognizer = build_recognizer(config.model)
     recognizer.cfg = config
 
     input_shape = (1, 8, 3, 32, 32)
     _do_test_2D_models(recognizer, target_layer_name, input_shape, 174)
 
-    config = get_cfg('recognition',
-                     'tpn/tpn_slowonly_r50_8x8x1_150e_kinetics_rgb.py')
+    config = get_recognizer_cfg(
+        'tpn/tpn_slowonly_r50_8x8x1_150e_kinetics_rgb.py')
     config.model['backbone']['pretrained'] = None
-    recognizer = build_recognizer(config.model, test_cfg=config.test_cfg)
+    recognizer = build_recognizer(config.model)
     recognizer.cfg = config
     input_shape = (1, 3, 3, 8, 32, 32)
     _do_test_3D_models(recognizer, target_layer_name, input_shape)
 
 
 def test_c3d():
-    config = get_cfg('recognition',
-                     'c3d/c3d_sports1m_16x1x1_45e_ucf101_rgb.py')
+    config = get_recognizer_cfg('c3d/c3d_sports1m_16x1x1_45e_ucf101_rgb.py')
     config.model['backbone']['pretrained'] = None
-    recognizer = build_recognizer(config.model, test_cfg=config.test_cfg)
+    recognizer = build_recognizer(config.model)
     recognizer.cfg = config
     input_shape = (1, 1, 3, 16, 112, 112)
     target_layer_name = 'backbone/conv5a/activate'
@@ -211,12 +207,12 @@ def test_c3d():
 @pytest.mark.skipif(
     not torch.cuda.is_available(), reason='requires CUDA support')
 def test_tin():
-    config = get_cfg('recognition',
-                     'tin/tin_tsm_finetune_r50_1x1x8_50e_kinetics400_rgb.py')
+    config = get_recognizer_cfg(
+        'tin/tin_tsm_finetune_r50_1x1x8_50e_kinetics400_rgb.py')
     config.model['backbone']['pretrained'] = None
     target_layer_name = 'backbone/layer4/1/relu'
 
-    recognizer = build_recognizer(config.model, test_cfg=config.test_cfg)
+    recognizer = build_recognizer(config.model)
     recognizer.cfg = config
     input_shape = (1, 8, 3, 64, 64)
     _do_test_2D_models(
@@ -224,10 +220,9 @@ def test_tin():
 
 
 def test_x3d():
-    config = get_cfg('recognition',
-                     'x3d/x3d_s_13x6x1_facebook_kinetics400_rgb.py')
+    config = get_recognizer_cfg('x3d/x3d_s_13x6x1_facebook_kinetics400_rgb.py')
     config.model['backbone']['pretrained'] = None
-    recognizer = build_recognizer(config.model, test_cfg=config.test_cfg)
+    recognizer = build_recognizer(config.model)
     recognizer.cfg = config
     input_shape = (1, 1, 3, 13, 32, 32)
     target_layer_name = 'backbone/layer4/1/relu'
diff --git a/tests/test_models/test_localizers/test_bmn.py b/tests/test_models/test_localizers/test_bmn.py
index 60c402fa62..7c650a9114 100644
--- a/tests/test_models/test_localizers/test_bmn.py
+++ b/tests/test_models/test_localizers/test_bmn.py
@@ -6,11 +6,11 @@
 
 
 def test_bmn():
-    model_cfg, _, _ = get_localizer_cfg(
+    model_cfg = get_localizer_cfg(
         'bmn/bmn_400x100_2x8_9e_activitynet_feature.py')
 
     if torch.cuda.is_available():
-        localizer_bmn = build_localizer(model_cfg).cuda()
+        localizer_bmn = build_localizer(model_cfg.model).cuda()
         raw_feature = torch.rand(8, 400, 100).cuda()
         gt_bbox = np.array([[[0.1, 0.3], [0.375, 0.625]]] * 8)
         losses = localizer_bmn(raw_feature, gt_bbox)
@@ -32,7 +32,7 @@ def test_bmn():
                 video_meta=video_meta,
                 return_loss=False)
     else:
-        localizer_bmn = build_localizer(model_cfg)
+        localizer_bmn = build_localizer(model_cfg.model)
         raw_feature = torch.rand(8, 400, 100)
         gt_bbox = torch.Tensor([[[0.1, 0.3], [0.375, 0.625]]] * 8)
         losses = localizer_bmn(raw_feature, gt_bbox)
diff --git a/tests/test_models/test_localizers/test_pem.py b/tests/test_models/test_localizers/test_pem.py
index deb053e6c6..a20e42dbf2 100644
--- a/tests/test_models/test_localizers/test_pem.py
+++ b/tests/test_models/test_localizers/test_pem.py
@@ -5,10 +5,10 @@
 
 
 def test_pem():
-    model_cfg, _, _ = get_localizer_cfg(
+    model_cfg = get_localizer_cfg(
         'bsn/bsn_pem_400x100_1x16_20e_activitynet_feature.py')
 
-    localizer_pem = build_localizer(model_cfg)
+    localizer_pem = build_localizer(model_cfg.model)
     bsp_feature = torch.rand(8, 100, 32)
     reference_temporal_iou = torch.rand(8, 100)
     losses = localizer_pem(bsp_feature, reference_temporal_iou)
diff --git a/tests/test_models/test_localizers/test_tem.py b/tests/test_models/test_localizers/test_tem.py
index bbe382839c..741ed76d22 100644
--- a/tests/test_models/test_localizers/test_tem.py
+++ b/tests/test_models/test_localizers/test_tem.py
@@ -5,10 +5,10 @@
 
 
 def test_tem():
-    model_cfg, _, _ = get_localizer_cfg(
+    model_cfg = get_localizer_cfg(
         'bsn/bsn_tem_400x100_1x16_20e_activitynet_feature.py')
 
-    localizer_tem = build_localizer(model_cfg)
+    localizer_tem = build_localizer(model_cfg.model)
     raw_feature = torch.rand(8, 400, 100)
     gt_bbox = torch.Tensor([[[1.0, 3.0], [3.0, 5.0]]] * 8)
     losses = localizer_tem(raw_feature, gt_bbox)
diff --git a/tests/test_models/test_recognizers/test_audio_recognizer.py b/tests/test_models/test_recognizers/test_audio_recognizer.py
index 3f4579da45..c2401063b1 100644
--- a/tests/test_models/test_recognizers/test_audio_recognizer.py
+++ b/tests/test_models/test_recognizers/test_audio_recognizer.py
@@ -5,12 +5,11 @@
 
 
 def test_audio_recognizer():
-    model, train_cfg, test_cfg = get_audio_recognizer_cfg(
+    config = get_audio_recognizer_cfg(
         'resnet/tsn_r18_64x1x1_100e_kinetics400_audio_feature.py')
-    model['backbone']['pretrained'] = None
+    config.model['backbone']['pretrained'] = None
 
-    recognizer = build_recognizer(
-        model, train_cfg=train_cfg, test_cfg=test_cfg)
+    recognizer = build_recognizer(config.model)
 
     input_shape = (1, 3, 1, 128, 80)
     demo_inputs = generate_recognizer_demo_inputs(
diff --git a/tests/test_models/test_recognizers/test_recognizer2d.py b/tests/test_models/test_recognizers/test_recognizer2d.py
index eaa539f175..95d23d9009 100644
--- a/tests/test_models/test_recognizers/test_recognizer2d.py
+++ b/tests/test_models/test_recognizers/test_recognizer2d.py
@@ -5,12 +5,10 @@
 
 
 def test_tsn():
-    model, train_cfg, test_cfg = get_recognizer_cfg(
-        'tsn/tsn_r50_1x1x3_100e_kinetics400_rgb.py')
-    model['backbone']['pretrained'] = None
+    config = get_recognizer_cfg('tsn/tsn_r50_1x1x3_100e_kinetics400_rgb.py')
+    config.model['backbone']['pretrained'] = None
 
-    recognizer = build_recognizer(
-        model, train_cfg=train_cfg, test_cfg=test_cfg)
+    recognizer = build_recognizer(config.model)
 
     input_shape = (1, 3, 3, 32, 32)
     demo_inputs = generate_recognizer_demo_inputs(input_shape)
@@ -34,12 +32,10 @@ def test_tsn():
 
 
 def test_tsm():
-    model, train_cfg, test_cfg = get_recognizer_cfg(
-        'tsm/tsm_r50_1x1x8_50e_kinetics400_rgb.py')
-    model['backbone']['pretrained'] = None
+    config = get_recognizer_cfg('tsm/tsm_r50_1x1x8_50e_kinetics400_rgb.py')
+    config.model['backbone']['pretrained'] = None
 
-    recognizer = build_recognizer(
-        model, train_cfg=train_cfg, test_cfg=test_cfg)
+    recognizer = build_recognizer(config.model)
 
     input_shape = (1, 8, 3, 32, 32)
     demo_inputs = generate_recognizer_demo_inputs(input_shape)
@@ -61,9 +57,8 @@ def test_tsm():
     demo_inputs = generate_recognizer_demo_inputs(input_shape)
     imgs = demo_inputs['imgs']
 
-    test_cfg = dict(average_clips='prob')
-    recognizer = build_recognizer(
-        model, train_cfg=train_cfg, test_cfg=test_cfg)
+    config.model.test_cfg = dict(average_clips='prob')
+    recognizer = build_recognizer(config.model)
 
     # Test forward test
     with torch.no_grad():
@@ -77,13 +72,48 @@ def test_tsm():
         recognizer(one_img, gradcam=True)
 
 
+def test_tpn():
+    config = get_recognizer_cfg('tpn/tpn_tsm_r50_1x1x8_150e_sthv1_rgb.py')
+    config.model['backbone']['pretrained'] = None
+
+    recognizer = build_recognizer(config.model)
+
+    input_shape = (1, 8, 3, 224, 224)
+    demo_inputs = generate_recognizer_demo_inputs(input_shape)
+
+    imgs = demo_inputs['imgs']
+    gt_labels = demo_inputs['gt_labels']
+
+    losses = recognizer(imgs, gt_labels)
+    assert isinstance(losses, dict)
+
+    # Test forward test
+    with torch.no_grad():
+        img_list = [img[None, :] for img in imgs]
+        for one_img in img_list:
+            recognizer(one_img, None, return_loss=False)
+
+    # Test forward gradcam
+    recognizer(imgs, gradcam=True)
+    for one_img in img_list:
+        recognizer(one_img, gradcam=True)
+
+    # Test forward dummy
+    with torch.no_grad():
+        _recognizer = build_recognizer(config.model)
+        img_list = [img[None, :] for img in imgs]
+        if hasattr(_recognizer, 'forward_dummy'):
+            _recognizer.forward = _recognizer.forward_dummy
+        for one_img in img_list:
+            _recognizer(one_img)
+
+
 def test_tanet():
-    model, train_cfg, test_cfg = get_recognizer_cfg(
+    config = get_recognizer_cfg(
         'tanet/tanet_r50_dense_1x1x8_100e_kinetics400_rgb.py')
-    model['backbone']['pretrained'] = None
+    config.model['backbone']['pretrained'] = None
 
-    recognizer = build_recognizer(
-        model, train_cfg=train_cfg, test_cfg=test_cfg)
+    recognizer = build_recognizer(config.model)
 
     input_shape = (1, 8, 3, 32, 32)
     demo_inputs = generate_recognizer_demo_inputs(input_shape)
@@ -105,9 +135,8 @@ def test_tanet():
     demo_inputs = generate_recognizer_demo_inputs(input_shape)
     imgs = demo_inputs['imgs']
 
-    test_cfg = dict(average_clips='prob')
-    recognizer = build_recognizer(
-        model, train_cfg=train_cfg, test_cfg=test_cfg)
+    config.model.test_cfg = dict(average_clips='prob')
+    recognizer = build_recognizer(config.model)
 
     # Test forward test
     with torch.no_grad():
diff --git a/tests/test_models/test_recognizers/test_recognizer3d.py b/tests/test_models/test_recognizers/test_recognizer3d.py
index 262303059b..e46543c5ed 100644
--- a/tests/test_models/test_recognizers/test_recognizer3d.py
+++ b/tests/test_models/test_recognizers/test_recognizer3d.py
@@ -5,13 +5,11 @@
 
 
 def test_i3d():
-    model, train_cfg, test_cfg = get_recognizer_cfg(
-        'i3d/i3d_r50_32x2x1_100e_kinetics400_rgb.py')
-    model['backbone']['pretrained2d'] = False
-    model['backbone']['pretrained'] = None
+    config = get_recognizer_cfg('i3d/i3d_r50_32x2x1_100e_kinetics400_rgb.py')
+    config.model['backbone']['pretrained2d'] = False
+    config.model['backbone']['pretrained'] = None
 
-    recognizer = build_recognizer(
-        model, train_cfg=train_cfg, test_cfg=test_cfg)
+    recognizer = build_recognizer(config.model)
 
     input_shape = (1, 3, 3, 8, 32, 32)
     demo_inputs = generate_recognizer_demo_inputs(input_shape, '3D')
@@ -56,14 +54,13 @@ def test_i3d():
 
 
 def test_r2plus1d():
-    model, train_cfg, test_cfg = get_recognizer_cfg(
+    config = get_recognizer_cfg(
         'r2plus1d/r2plus1d_r34_8x8x1_180e_kinetics400_rgb.py')
-    model['backbone']['pretrained2d'] = False
-    model['backbone']['pretrained'] = None
-    model['backbone']['norm_cfg'] = dict(type='BN3d')
+    config.model['backbone']['pretrained2d'] = False
+    config.model['backbone']['pretrained'] = None
+    config.model['backbone']['norm_cfg'] = dict(type='BN3d')
 
-    recognizer = build_recognizer(
-        model, train_cfg=train_cfg, test_cfg=test_cfg)
+    recognizer = build_recognizer(config.model)
 
     input_shape = (1, 3, 3, 8, 32, 32)
     demo_inputs = generate_recognizer_demo_inputs(input_shape, '3D')
@@ -107,11 +104,10 @@ def test_r2plus1d():
 
 
 def test_slowfast():
-    model, train_cfg, test_cfg = get_recognizer_cfg(
+    config = get_recognizer_cfg(
         'slowfast/slowfast_r50_4x16x1_256e_kinetics400_rgb.py')
 
-    recognizer = build_recognizer(
-        model, train_cfg=train_cfg, test_cfg=test_cfg)
+    recognizer = build_recognizer(config.model)
 
     input_shape = (1, 3, 3, 16, 32, 32)
     demo_inputs = generate_recognizer_demo_inputs(input_shape, '3D')
@@ -154,9 +150,8 @@ def test_slowfast():
             recognizer(one_img, gradcam=True)
 
         # Test the feature max_testing_views
-        test_cfg['max_testing_views'] = 1
-        recognizer = build_recognizer(
-            model, train_cfg=train_cfg, test_cfg=test_cfg)
+        config.model.test_cfg['max_testing_views'] = 1
+        recognizer = build_recognizer(config.model)
         with torch.no_grad():
             img_list = [img[None, :] for img in imgs]
             for one_img in img_list:
@@ -164,13 +159,12 @@ def test_slowfast():
 
 
 def test_csn():
-    model, train_cfg, test_cfg = get_recognizer_cfg(
+    config = get_recognizer_cfg(
         'csn/ircsn_ig65m_pretrained_r152_32x2x1_58e_kinetics400_rgb.py')
-    model['backbone']['pretrained2d'] = False
-    model['backbone']['pretrained'] = None
+    config.model['backbone']['pretrained2d'] = False
+    config.model['backbone']['pretrained'] = None
 
-    recognizer = build_recognizer(
-        model, train_cfg=train_cfg, test_cfg=test_cfg)
+    recognizer = build_recognizer(config.model)
 
     input_shape = (1, 3, 3, 8, 32, 32)
     demo_inputs = generate_recognizer_demo_inputs(input_shape, '3D')
@@ -214,15 +208,14 @@ def test_csn():
 
 
 def test_tpn():
-    model, train_cfg, test_cfg = get_recognizer_cfg(
-        'tpn/tpn_tsm_r50_1x1x8_150e_sthv1_rgb.py')
-    model['backbone']['pretrained'] = None
+    config = get_recognizer_cfg(
+        'tpn/tpn_slowonly_r50_8x8x1_150e_kinetics_rgb.py')
+    config.model['backbone']['pretrained'] = None
 
-    recognizer = build_recognizer(
-        model, train_cfg=train_cfg, test_cfg=test_cfg)
+    recognizer = build_recognizer(config.model)
 
-    input_shape = (1, 8, 3, 224, 224)
-    demo_inputs = generate_recognizer_demo_inputs(input_shape)
+    input_shape = (1, 8, 3, 1, 224, 224)
+    demo_inputs = generate_recognizer_demo_inputs(input_shape, '3D')
 
     imgs = demo_inputs['imgs']
     gt_labels = demo_inputs['gt_labels']
@@ -236,62 +229,26 @@ def test_tpn():
         for one_img in img_list:
             recognizer(one_img, None, return_loss=False)
 
-    # Test forward dummy
-    with torch.no_grad():
-        img_list = [img[None, :] for img in imgs]
-        if hasattr(model, 'forward_dummy'):
-            model.forward = model.forward_dummy
-        for one_img in img_list:
-            recognizer(one_img, None, return_loss=False)
-
     # Test forward gradcam
     recognizer(imgs, gradcam=True)
     for one_img in img_list:
         recognizer(one_img, gradcam=True)
 
-    model, train_cfg, test_cfg = get_recognizer_cfg(
-        'tpn/tpn_slowonly_r50_8x8x1_150e_kinetics_rgb.py')
-    model['backbone']['pretrained'] = None
-
-    recognizer = build_recognizer(
-        model, train_cfg=train_cfg, test_cfg=test_cfg)
-
-    input_shape = (1, 8, 3, 1, 224, 224)
-    demo_inputs = generate_recognizer_demo_inputs(input_shape, '3D')
-
-    imgs = demo_inputs['imgs']
-    gt_labels = demo_inputs['gt_labels']
-
-    losses = recognizer(imgs, gt_labels)
-    assert isinstance(losses, dict)
-
-    # Test forward test
-    with torch.no_grad():
-        img_list = [img[None, :] for img in imgs]
-        for one_img in img_list:
-            recognizer(one_img, None, return_loss=False)
-
     # Test dummy forward
     with torch.no_grad():
+        _recognizer = build_recognizer(config.model)
         img_list = [img[None, :] for img in imgs]
-        if hasattr(model, 'forward_dummy'):
-            model.forward = model.forward_dummy
+        if hasattr(_recognizer, 'forward_dummy'):
+            _recognizer.forward = _recognizer.forward_dummy
         for one_img in img_list:
-            recognizer(one_img, None, return_loss=False)
-
-    # Test forward gradcam
-    recognizer(imgs, gradcam=True)
-    for one_img in img_list:
-        recognizer(one_img, gradcam=True)
+            _recognizer(one_img)
 
 
 def test_c3d():
-    model, train_cfg, test_cfg = get_recognizer_cfg(
-        'c3d/c3d_sports1m_16x1x1_45e_ucf101_rgb.py')
-    model['backbone']['pretrained'] = None
+    config = get_recognizer_cfg('c3d/c3d_sports1m_16x1x1_45e_ucf101_rgb.py')
+    config.model['backbone']['pretrained'] = None
 
-    recognizer = build_recognizer(
-        model, train_cfg=train_cfg, test_cfg=test_cfg)
+    recognizer = build_recognizer(config.model)
 
     input_shape = (1, 3, 3, 16, 112, 112)
     demo_inputs = generate_recognizer_demo_inputs(input_shape, '3D')
diff --git a/tests/test_runtime/test_config.py b/tests/test_runtime/test_config.py
index 7045b528d9..e11a334bef 100644
--- a/tests/test_runtime/test_config.py
+++ b/tests/test_runtime/test_config.py
@@ -40,10 +40,7 @@ def test_config_build_recognizer():
         if 'pretrained' in config_mod.model['backbone']:
             config_mod.model['backbone']['pretrained'] = None
 
-        recognizer = build_recognizer(
-            config_mod.model,
-            train_cfg=config_mod.train_cfg,
-            test_cfg=config_mod.test_cfg)
+        recognizer = build_recognizer(config_mod.model)
         assert isinstance(recognizer, nn.Module)
 
 
diff --git a/tools/test.py b/tools/test.py
index 3ba34eb81c..4fc0d32522 100644
--- a/tools/test.py
+++ b/tools/test.py
@@ -171,7 +171,8 @@ def main():
     data_loader = build_dataloader(dataset, **dataloader_setting)
 
     # build the model and load checkpoint
-    model = build_model(cfg.model, train_cfg=None, test_cfg=cfg.test_cfg)
+    model = build_model(
+        cfg.model, train_cfg=None, test_cfg=cfg.get('test_cfg'))
 
     register_module_hooks(model.backbone, cfg.module_hooks)