Repository: open-mmlab/mmaction2

Issue:
Refactor evaluation of ava
Currently `metrics` in ava evaluation are `'proposal', 'bbox'`, which are not metrics. https://github.com/open-mmlab/mmaction2/blob/master/mmaction/datasets/ava_dataset.py#L319 .

We need to change metrics to `mAP, Recall` etc, and make `proposal/bbox` as metric_options 


Discussion:

Resolving patch:
diff --git a/mmaction/core/evaluation/ava_utils.py b/mmaction/core/evaluation/ava_utils.py
--- a/mmaction/core/evaluation/ava_utils.py
+++ b/mmaction/core/evaluation/ava_utils.py
@@ -164,7 +164,7 @@ def ava_eval(result_file,
              max_dets=(100, ),
              verbose=True):
 
-    assert result_type in ['proposal', 'bbox']
+    assert result_type in ['mAP']
 
     start = time.time()
     categories, class_whitelist = read_labelmap(open(label_file))
@@ -213,7 +213,7 @@ def ava_eval(result_file,
             ret[f'AR@{num}'] = ar[i]
         return ret
 
-    if result_type == 'bbox':
+    if result_type == 'mAP':
         pascal_evaluator = det_eval.PascalDetectionEvaluator(categories)
 
         start = time.time()
diff --git a/mmaction/datasets/ava_dataset.py b/mmaction/datasets/ava_dataset.py
--- a/mmaction/datasets/ava_dataset.py
+++ b/mmaction/datasets/ava_dataset.py
@@ -316,10 +316,14 @@ def dump_results(self, results, out):
 
     def evaluate(self,
                  results,
-                 metrics=('proposal', 'bbox'),
+                 metrics=('mAP', ),
                  metric_options=None,
                  logger=None):
         # need to create a temp result file
+        assert len(metrics) == 1 and metrics[0] == 'mAP', (
+            'For evaluation on AVADataset, you need to use metrics "mAP" '
+            'See https://github.com/open-mmlab/mmaction2/pull/567 '
+            'for more info.')
         time_now = datetime.now().strftime('%Y%m%d_%H%M%S')
         temp_file = f'AVA_{time_now}_result.csv'
         results2csv(self, results, temp_file)


Test patch:
diff --git a/tests/test_data/test_datasets/test_ava_dataset.py b/tests/test_data/test_datasets/test_ava_dataset.py
--- a/tests/test_data/test_datasets/test_ava_dataset.py
+++ b/tests/test_data/test_datasets/test_ava_dataset.py
@@ -166,6 +166,4 @@ def test_ava_evaluate(self):
                                      [0.106, 0.445, 0.782, 0.673, 0.367]])
                        ]]
         res = ava_dataset.evaluate(fake_result)
-        assert_array_almost_equal(res['Recall@0.5@100'], 0.33333333)
-        assert_array_almost_equal(res['AR@100'], 0.15833333)
         assert_array_almost_equal(res['mAP@0.5IOU'], 0.027777778)
diff --git a/tests/test_metrics/test_accuracy.py b/tests/test_metrics/test_accuracy.py
--- a/tests/test_metrics/test_accuracy.py
+++ b/tests/test_metrics/test_accuracy.py
@@ -77,13 +77,8 @@ def test_ava_detection():
     result_path = osp.join(data_prefix, 'pred.csv')
     label_map = osp.join(data_prefix, 'action_list.txt')
 
-    # eval proposal
-    detection = ava_eval(result_path, 'proposal', label_map, gt_path, None)
-    assert_array_almost_equal(detection['Recall@0.5@100'], 0.41666667)
-    assert_array_almost_equal(detection['AR@100'], 0.08333333)
-
     # eval bbox
-    detection = ava_eval(result_path, 'bbox', label_map, gt_path, None)
+    detection = ava_eval(result_path, 'mAP', label_map, gt_path, None)
     assert_array_almost_equal(detection['mAP@0.5IOU'], 0.09385522)