Repository: open-mmlab/mmaction2

Issue:
About default value of `save_best` in EvalHook
By default `save_best=None`, which disables saving best eval ckpt.

I think saving best ckpt is a basic function during training. It doesn't require a lot of resources and should be enabled by default.

It could be done by either set `save_best='auto'` or add `evalution=dict(save_best='xxx')` in all configs. I prefer the latter.


Discussion:
I could create a pr to solve this. Waht do you think of this modification? @dreamerlin @innerlee 
`save_best='auto'` may be better, what do you think? @innerlee 
> `save_best='auto'` may be better, what do you think? @innerlee

Can user choose number of top best checkpoints to save？
> > `save_best='auto'` may be better, what do you think? @innerlee
> 
> Can user choose number of top best checkpoints to save？

No. But you can save last 5 epochs by `checkpoint_config = dict(interval=1, max_keep_ckpts =5)`

Resolving patch:
diff --git a/mmaction/core/evaluation/eval_hooks.py b/mmaction/core/evaluation/eval_hooks.py
--- a/mmaction/core/evaluation/eval_hooks.py
+++ b/mmaction/core/evaluation/eval_hooks.py
@@ -35,12 +35,10 @@ class EvalHook(Hook):
             ``mean_average_precision``, ``mmit_mean_average_precision``
             for action recognition dataset (RawframeDataset and VideoDataset).
             ``AR@AN``, ``auc`` for action localization dataset.
-            (ActivityNetDataset). ``Recall@0.5@100``, ``AR@100``,
-            ``mAP@0.5IOU`` for spatio-temporal action detection dataset
-            (AVADataset). If ``save_best`` is ``auto``, the first key
-             of the returned ``OrderedDict`` result will be used. The interval
-             of ``EvalHook`` should be divisible by that of ``CheckpointHook``.
-             Default: 'top1_acc'.
+            (ActivityNetDataset). ``mAP@0.5IOU`` for spatio-temporal action
+            detection dataset (AVADataset). If ``save_best`` is ``auto``, the
+            first key of the returned ``OrderedDict`` result will be used.
+            Default: 'auto'.
         rule (str | None, optional): Comparison rule for best score. If set to
             None, it will infer a reasonable rule. Keys such as 'acc', 'top'
             .etc will be inferred by 'greater' rule. Keys contain 'loss' will
@@ -60,7 +58,7 @@ def __init__(self,
                  start=None,
                  interval=1,
                  by_epoch=True,
-                 save_best=None,
+                 save_best='auto',
                  rule=None,
                  **eval_kwargs):
         if 'key_indicator' in eval_kwargs:
@@ -274,10 +272,10 @@ class DistEvalHook(EvalHook):
             ``mean_average_precision``, ``mmit_mean_average_precision``
             for action recognition dataset (RawframeDataset and VideoDataset).
             ``AR@AN``, ``auc`` for action localization dataset
-            (ActivityNetDataset). If ``save_best`` is ``auto``, the first key
-            of the returned ``OrderedDict`` result will be used. The interval
-            of ``EvalHook`` should be divisible of that in ``CheckpointHook``.
-            Default: None.
+            (ActivityNetDataset). ``mAP@0.5IOU`` for spatio-temporal action
+            detection dataset (AVADataset). If ``save_best`` is ``auto``, the
+            first key of the returned ``OrderedDict`` result will be used.
+            Default: 'auto'.
         rule (str | None, optional): Comparison rule for best score. If set to
             None, it will infer a reasonable rule. Keys such as 'acc', 'top'
             .etc will be inferred by 'greater' rule. Keys contain 'loss' will
@@ -296,7 +294,7 @@ def __init__(self,
                  start=None,
                  interval=1,
                  by_epoch=True,
-                 save_best=None,
+                 save_best='auto',
                  rule=None,
                  tmpdir=None,
                  gpu_collect=False,


Test patch:
diff --git a/tests/test_runtime/test_eval_hook.py b/tests/test_runtime/test_eval_hook.py
--- a/tests/test_runtime/test_eval_hook.py
+++ b/tests/test_runtime/test_eval_hook.py
@@ -93,7 +93,7 @@ def test_eval_hook():
         EvalHook(data_loader)
 
     with pytest.raises(ValueError):
-        # key_indicator must be valid when rule_map is None
+        # save_best must be valid when rule_map is None
         test_dataset = ExampleDataset()
         data_loader = DataLoader(test_dataset)
         EvalHook(data_loader, save_best='unsupport')
@@ -260,7 +260,8 @@ def test_start_param(EvalHookParam, _build_demo_runner, by_epoch):
 
     # 1. start=None, interval=1: perform evaluation after each epoch.
     runner = _build_demo_runner()
-    evalhook = EvalHookParam(dataloader, interval=1, by_epoch=by_epoch)
+    evalhook = EvalHookParam(
+        dataloader, interval=1, by_epoch=by_epoch, save_best=None)
     evalhook.evaluate = MagicMock()
     runner.register_hook(evalhook)
     runner.run([dataloader], [('train', 1)], 2)
@@ -269,7 +270,7 @@ def test_start_param(EvalHookParam, _build_demo_runner, by_epoch):
     # 2. start=1, interval=1: perform evaluation after each epoch.
     runner = _build_demo_runner()
     evalhook = EvalHookParam(
-        dataloader, start=1, interval=1, by_epoch=by_epoch)
+        dataloader, start=1, interval=1, by_epoch=by_epoch, save_best=None)
     evalhook.evaluate = MagicMock()
     runner.register_hook(evalhook)
     runner.run([dataloader], [('train', 1)], 2)
@@ -277,7 +278,8 @@ def test_start_param(EvalHookParam, _build_demo_runner, by_epoch):
 
     # 3. start=None, interval=2: perform evaluation after epoch 2, 4, 6, etc
     runner = _build_demo_runner()
-    evalhook = EvalHookParam(dataloader, interval=2, by_epoch=by_epoch)
+    evalhook = EvalHookParam(
+        dataloader, interval=2, by_epoch=by_epoch, save_best=None)
     evalhook.evaluate = MagicMock()
     runner.register_hook(evalhook)
     runner.run([dataloader], [('train', 1)], 2)
@@ -286,7 +288,7 @@ def test_start_param(EvalHookParam, _build_demo_runner, by_epoch):
     # 4. start=1, interval=2: perform evaluation after epoch 1, 3, 5, etc
     runner = _build_demo_runner()
     evalhook = EvalHookParam(
-        dataloader, start=1, interval=2, by_epoch=by_epoch)
+        dataloader, start=1, interval=2, by_epoch=by_epoch, save_best=None)
     evalhook.evaluate = MagicMock()
     runner.register_hook(evalhook)
     runner.run([dataloader], [('train', 1)], 3)
@@ -295,7 +297,8 @@ def test_start_param(EvalHookParam, _build_demo_runner, by_epoch):
     # 5. start=0/negative, interval=1: perform evaluation after each epoch and
     #    before epoch 1.
     runner = _build_demo_runner()
-    evalhook = EvalHookParam(dataloader, start=0, by_epoch=by_epoch)
+    evalhook = EvalHookParam(
+        dataloader, start=0, by_epoch=by_epoch, save_best=None)
     evalhook.evaluate = MagicMock()
     runner.register_hook(evalhook)
     runner.run([dataloader], [('train', 1)], 2)
@@ -303,7 +306,8 @@ def test_start_param(EvalHookParam, _build_demo_runner, by_epoch):
 
     runner = _build_demo_runner()
     with pytest.warns(UserWarning):
-        evalhook = EvalHookParam(dataloader, start=-2, by_epoch=by_epoch)
+        evalhook = EvalHookParam(
+            dataloader, start=-2, by_epoch=by_epoch, save_best=None)
     evalhook.evaluate = MagicMock()
     runner.register_hook(evalhook)
     runner.run([dataloader], [('train', 1)], 2)
@@ -312,7 +316,8 @@ def test_start_param(EvalHookParam, _build_demo_runner, by_epoch):
     # 6. resuming from epoch i, start = x (x<=i), interval =1: perform
     #    evaluation after each epoch and before the first epoch.
     runner = _build_demo_runner()
-    evalhook = EvalHookParam(dataloader, start=1, by_epoch=by_epoch)
+    evalhook = EvalHookParam(
+        dataloader, start=1, by_epoch=by_epoch, save_best=None)
     evalhook.evaluate = MagicMock()
     runner.register_hook(evalhook)
     if by_epoch:
@@ -325,7 +330,8 @@ def test_start_param(EvalHookParam, _build_demo_runner, by_epoch):
     # 7. resuming from epoch i, start = i+1/None, interval =1: perform
     #    evaluation after each epoch.
     runner = _build_demo_runner()
-    evalhook = EvalHookParam(dataloader, start=2, by_epoch=by_epoch)
+    evalhook = EvalHookParam(
+        dataloader, start=2, by_epoch=by_epoch, save_best=None)
     evalhook.evaluate = MagicMock()
     runner.register_hook(evalhook)
     if by_epoch:
diff --git a/tests/test_runtime/test_train.py b/tests/test_runtime/test_train.py
--- a/tests/test_runtime/test_train.py
+++ b/tests/test_runtime/test_train.py
@@ -73,7 +73,7 @@ def test_train_model():
         load_from=None,
         workflow=[('train', 1)],
         total_epochs=5,
-        evaluation=dict(interval=1, key_indicator='acc'),
+        evaluation=dict(interval=1, save_best='acc'),
         data=dict(
             videos_per_gpu=1,
             workers_per_gpu=0,