Repository: deepchecks/deepchecks

Issue:
[FEAT][CV] Mean average precision metric
**Describe alternatives you've considered**
The mean average precision (mAP) is one of the main metrics in object detection. I think this check would be really informative in terms of measuring the performance of the detector. This check should return the following results:
-  AP@[.5 : .95] (Average precision over 10 different thresholds from 0.5 to 0.95 incrementing by 0.05)
- Graph with the AP over all classes for every IoU threshold
- AP for different object dimensions:
  - for small objects (IoU range 0<x<32^2]
  - for medium objects (IoU range 32^2<x<96^2)
  - for large objects (IoU range x>96^2)

Please see the COCO evaluation metrics for more information



**Additional context**
Add any other context or screenshots about the feature request here.



Discussion:

Resolving patch:
diff --git a/deepchecks/vision/checks/performance/__init__.py b/deepchecks/vision/checks/performance/__init__.py
--- a/deepchecks/vision/checks/performance/__init__.py
+++ b/deepchecks/vision/checks/performance/__init__.py
@@ -10,9 +10,11 @@
 #
 """Module containing the performance check in the vision package."""
 from .class_performance import ClassPerformance
+from .mean_average_precision_report import MeanAveragePrecisionReport
 from .robustness_report import RobustnessReport
 
 __all__ = [
     "ClassPerformance",
-    "RobustnessReport"
+    "MeanAveragePrecisionReport",
+    "RobustnessReport",
 ]
diff --git a/deepchecks/vision/checks/performance/mean_average_precision_report.py b/deepchecks/vision/checks/performance/mean_average_precision_report.py
new file mode 100644
--- /dev/null
+++ b/deepchecks/vision/checks/performance/mean_average_precision_report.py
@@ -0,0 +1,115 @@
+# ----------------------------------------------------------------------------
+# Copyright (C) 2021-2022 Deepchecks (https://www.deepchecks.com)
+#
+# This file is part of Deepchecks.
+# Deepchecks is distributed under the terms of the GNU Affero General
+# Public License (version 3 or later).
+# You should have received a copy of the GNU Affero General Public License
+# along with Deepchecks.  If not, see <http://www.gnu.org/licenses/>.
+# ----------------------------------------------------------------------------
+#
+"""Module containing mean average precision report check."""
+from collections import defaultdict
+import math
+from typing import TypeVar, Tuple, Any
+
+import pandas as pd
+import plotly.express as px
+import numpy as np
+
+from deepchecks.core import CheckResult, ConditionResult, DatasetKind
+from deepchecks.utils.strings import format_number
+from deepchecks.vision import SingleDatasetCheck, Context
+from deepchecks.vision.dataset import TaskType
+from deepchecks.vision.metrics_utils.detection_precision_recall import AveragePrecision
+
+__all__ = ['MeanAveragePrecisionReport']
+
+MPR = TypeVar('MPR', bound='MeanAveragePrecisionReport')
+
+
+class MeanAveragePrecisionReport(SingleDatasetCheck):
+    """Summarize mean average precision metrics on a dataset and model per IoU and area range.
+
+    Parameters
+    ----------
+    area_range: tuple, default: (32**2, 96**2)
+        Slices for small/medium/large buckets.
+    """
+
+    def __init__(self, area_range: Tuple = (32**2, 96**2)):
+        super().__init__()
+        self._ap_metric = AveragePrecision(return_option=None, area_range=area_range)
+
+    def initialize_run(self, context: Context, dataset_kind: DatasetKind = None):
+        """Initialize run by asserting task type and initializing metric."""
+        context.assert_task_type(TaskType.OBJECT_DETECTION)
+
+    def update(self, context: Context, batch: Any, dataset_kind: DatasetKind):
+        """Update the metrics by passing the batch to ignite metric update method."""
+        dataset = context.get_data_by_kind(dataset_kind)
+        images = batch[0]
+        label = dataset.label_transformer(batch[1])
+        prediction = context.prediction_formatter(context.infer(images))
+        self._ap_metric.update((prediction, label))
+
+    def compute(self, context: Context, dataset_kind: DatasetKind) -> CheckResult:
+        """Compute the metric result using the ignite metrics compute method and create display."""
+        small_area = int(math.sqrt(self._ap_metric.area_range[0]))
+        large_area = int(math.sqrt(self._ap_metric.area_range[1]))
+        res = self._ap_metric.compute()[0]['precision']
+        rows = []
+        for title, area_name in zip(['All',
+                                     f'Small (area < {small_area}^2)',
+                                     f'Medium ({small_area}^2 < area < {large_area}^2)',
+                                     f'Large (area < {large_area}^2)'],
+                                    ['all', 'small', 'medium', 'large']):
+            area_scores = [title]
+            area_scores.append(self._ap_metric.get_classes_scores_at(res, area=area_name, max_dets=100))
+            area_scores.append(self._ap_metric.get_classes_scores_at(res, iou=0.5, area=area_name, max_dets=100))
+            area_scores.append(self._ap_metric.get_classes_scores_at(res, iou=0.75, area=area_name, max_dets=100))
+
+            rows.append(area_scores)
+
+        results = pd.DataFrame(columns=['Area size', 'mAP@0.5..0.95 (%)', 'AP@.50 (%)', 'AP@.75 (%)'])
+        for i in range(len(rows)):
+            results.loc[i] = rows[i]
+        results = results.set_index('Area size')
+
+        filtered_res = self._ap_metric.filter_res(res, area='all', max_dets=100)
+        filtered_res_shape = filtered_res.shape
+        filtered_res = np.reshape(filtered_res, (filtered_res_shape[0], filtered_res_shape[3]))
+        mean_res = np.zeros(filtered_res_shape[0])
+        for i in range(filtered_res_shape[0]):
+            mean_res[i] = np.mean(filtered_res[i][filtered_res[i] > -1])
+
+        data = {
+            'IoU': self._ap_metric.iou_thresholds,
+            'AP (%)': mean_res
+        }
+        df = pd.DataFrame.from_dict(data)
+
+        fig = px.line(df, x='IoU', y='AP (%)', title='Mean Average Precision over increasing IoU thresholds')
+
+        return CheckResult(value=results, display=[results, fig])
+
+    def add_condition_test_average_precision_not_less_than(self: MPR, min_score: float) -> MPR:
+        """Add condition - mAP score is not less than given score.
+
+        Parameters
+        ----------
+        min_score : float
+            Minimum score to pass the check.
+        """
+        def condition(df: pd.DataFrame):
+            not_passed = defaultdict(dict)
+            for index, column in zip(df.index, df.columns):
+                if df.loc[index, column] < min_score:
+                    not_passed[index][column] = format_number(df.loc[index, column], 3)
+            if len(not_passed):
+                details = f'Found scores below threshold:\n' \
+                          f'{dict(not_passed)}'
+                return ConditionResult(False, details)
+            return ConditionResult(True)
+
+        return self.add_condition(f'Scores are not less than {min_score}', condition)
diff --git a/deepchecks/vision/metrics_utils/detection_precision_recall.py b/deepchecks/vision/metrics_utils/detection_precision_recall.py
--- a/deepchecks/vision/metrics_utils/detection_precision_recall.py
+++ b/deepchecks/vision/metrics_utils/detection_precision_recall.py
@@ -116,15 +116,15 @@ def compute(self):
                     reses["precision"][iou_i, area_i, dets_i] = np.array(precision_list)
                     reses["recall"][iou_i, area_i, dets_i] = np.array(recall_list)
         if self.return_option == 0:
-            return torch.tensor(self.get_val_at(reses["precision"],
-                                                max_dets=self.max_dets[0],
-                                                area=self.area_ranges_names[0],
-                                                get_mean_val=False))
+            return torch.tensor(self.get_classes_scores_at(reses["precision"],
+                                                           max_dets=self.max_dets[0],
+                                                           area=self.area_ranges_names[0],
+                                                           get_mean_val=False))
         elif self.return_option == 1:
-            return torch.tensor(self.get_val_at(reses["recall"],
-                                                max_dets=self.max_dets[0],
-                                                area=self.area_ranges_names[0],
-                                                get_mean_val=False))
+            return torch.tensor(self.get_classes_scores_at(reses["recall"],
+                                                           max_dets=self.max_dets[0],
+                                                           area=self.area_ranges_names[0],
+                                                           get_mean_val=False))
         return [reses]
 
     def _group_detections(self, dt, gt):
@@ -270,9 +270,8 @@ def _is_ignore_area(self, area_bb, area_size):
             return not area_bb > self.area_range[1]
         return False
 
-    def get_val_at(self, res: np.array, iou: float = None, area: str = None, max_dets: int = None,
-                   get_mean_val: bool = True, zeroed_negative: bool = True):
-        """Get the value a result by the filtering values.
+    def filter_res(self, res: np.array, iou: float = None, area: str = None, max_dets: int = None):
+        """Get the value of a result by the filtering values.
 
         Parameters
         ----------
@@ -284,15 +283,11 @@ def get_val_at(self, res: np.array, iou: float = None, area: str = None, max_det
             filter by are range name ["small", "medium", "large", "all"]
         max_dets : int, default: None
             filter by max detections
-        get_mean_val : bool, default: True
-            get mean value if True, if False get per class
-        zeroed_negative : bool, default: True
-            if getting the class results list set negative (-1) values to 0
 
         Returns
         -------
-        Union[List[float], float]
-           The mean value of the classes scores or the scores list.
+        np.array
+           The filtered result.
         """
         if iou:
             iou_i = [i for i, iou_thres in enumerate(self.iou_thresholds) if iou == iou_thres]
@@ -303,6 +298,33 @@ def get_val_at(self, res: np.array, iou: float = None, area: str = None, max_det
         if max_dets:
             dets_i = [i for i, det in enumerate(self.max_dets) if max_dets == det]
             res = res[:, :, dets_i, :]
+        return res
+
+    def get_classes_scores_at(self, res: np.array, iou: float = None, area: str = None, max_dets: int = None,
+                              get_mean_val: bool = True, zeroed_negative: bool = True):
+        """Get the mean value of the classes scores and the result values.
+
+        Parameters
+        ----------
+        res: np.array
+            either prrecision or recall when using the '2' return option
+        iou : float, default: None
+            filter by iou threshold
+        area : str, default: None
+            filter by are range name ["small", "medium", "large", "all"]
+        max_dets : int, default: None
+            filter by max detections
+        get_mean_val : bool, default: True
+            get mean value if True, if False get per class
+        zeroed_negative : bool, default: True
+            if getting the class results list set negative (-1) values to 0
+
+        Returns
+        -------
+        Union[List[float], float]
+           The mean value of the classes scores or the scores list.
+        """
+        res = self.filter_res(res, iou, area, max_dets)
         res = np.mean(res[:, :, :], axis=0)
         if get_mean_val:
             return np.mean(res[res > -1])


Test patch:
diff --git a/tests/vision/checks/performance/class_performance_test.py b/tests/vision/checks/performance/class_performance_test.py
--- a/tests/vision/checks/performance/class_performance_test.py
+++ b/tests/vision/checks/performance/class_performance_test.py
@@ -15,8 +15,7 @@
 
 import torch.nn as nn
 
-from hamcrest import assert_that, has_entries, close_to, equal_to, is_in
-from tests.vision.vision_conftest import *
+from hamcrest import assert_that, close_to, equal_to, is_in
 
 
 def test_mnist_largest(mnist_dataset_train, mnist_dataset_test, trained_mnist):
@@ -113,4 +112,4 @@ def test_class_list(mnist_dataset_train, mnist_dataset_test, trained_mnist):
     assert_that(result.value['Class'].iloc[0], equal_to(1))
     assert_that(result.value['Class'].iloc[1], equal_to(1))
     assert_that(result.value['Class'].iloc[2], equal_to(1))
-    assert_that(result.value['Class'].iloc[3], equal_to(1))
\ No newline at end of file
+    assert_that(result.value['Class'].iloc[3], equal_to(1))
diff --git a/tests/vision/checks/performance/mean_average_precision_report_test.py b/tests/vision/checks/performance/mean_average_precision_report_test.py
new file mode 100644
--- /dev/null
+++ b/tests/vision/checks/performance/mean_average_precision_report_test.py
@@ -0,0 +1,100 @@
+# ----------------------------------------------------------------------------
+# Copyright (C) 2021 Deepchecks (https://www.deepchecks.com)
+#
+# This file is part of Deepchecks.
+# Deepchecks is distributed under the terms of the GNU Affero General
+# Public License (version 3 or later).
+# You should have received a copy of the GNU Affero General Public License
+# along with Deepchecks.  If not, see <http://www.gnu.org/licenses/>.
+# ----------------------------------------------------------------------------
+#
+import re
+from hamcrest import assert_that, close_to, has_length, calling, raises
+
+from tests.checks.utils import equal_condition_result
+from deepchecks.core.errors import ModelValidationError
+from deepchecks.vision.checks.performance import MeanAveragePrecisionReport
+from deepchecks.vision.datasets.detection.coco import yolo_prediction_formatter
+from deepchecks.vision.utils import DetectionPredictionFormatter
+
+def test_mnist_error(mnist_dataset_test, trained_mnist):
+    # Arrange
+    check = MeanAveragePrecisionReport()
+    # Act
+    assert_that(
+        calling(check.run).with_args(mnist_dataset_test, trained_mnist),
+            raises(ModelValidationError, r'Check is irrelevant for task of type TaskType.CLASSIFICATION')
+    )
+
+
+def test_coco(coco_test_visiondata, trained_yolov5_object_detection):
+    # Arrange
+    pred_formatter = DetectionPredictionFormatter(yolo_prediction_formatter)
+    check = MeanAveragePrecisionReport() \
+            .add_condition_test_average_precision_not_less_than(0.1) \
+            .add_condition_test_average_precision_not_less_than(0.4)
+
+    # Act
+    result = check.run(coco_test_visiondata,
+                       trained_yolov5_object_detection, prediction_formatter=pred_formatter)
+
+    # Assert
+    df = result.value
+    assert_that(df, has_length(4))
+
+    assert_that(df.loc['All', 'mAP@0.5..0.95 (%)'], close_to(0.361, 0.001))
+    assert_that(df.loc['All', 'AP@.50 (%)'], close_to(0.501, 0.001))
+    assert_that(df.loc['All', 'AP@.75 (%)'], close_to(0.376, 0.001))
+
+    assert_that(df.loc['Small (area < 32^2)', 'mAP@0.5..0.95 (%)'], close_to(0.188, 0.001))
+    assert_that(df.loc['Small (area < 32^2)', 'AP@.50 (%)'], close_to(0.288, 0.001))
+    assert_that(df.loc['Small (area < 32^2)', 'AP@.75 (%)'], close_to(0.193, 0.001))
+
+    assert_that(df.loc['Medium (32^2 < area < 96^2)', 'mAP@0.5..0.95 (%)'], close_to(0.367, 0.001))
+    assert_that(df.loc['Medium (32^2 < area < 96^2)', 'AP@.50 (%)'], close_to(0.568, 0.001))
+    assert_that(df.loc['Medium (32^2 < area < 96^2)', 'AP@.75 (%)'], close_to(0.369, 0.001))
+
+    assert_that(df.loc['Large (area < 96^2)', 'mAP@0.5..0.95 (%)'], close_to(0.476, 0.001))
+    assert_that(df.loc['Large (area < 96^2)', 'AP@.50 (%)'], close_to(0.575, 0.001))
+    assert_that(df.loc['Large (area < 96^2)', 'AP@.75 (%)'], close_to(0.533, 0.001))
+
+    assert_that(result.conditions_results[0], equal_condition_result(
+        is_pass=True,
+        name='Scores are not less than 0.1'
+    ))
+
+    assert_that(result.conditions_results[1], equal_condition_result(
+        is_pass=False,
+        name='Scores are not less than 0.4',
+        details="Found scores below threshold:\n{'All': {'mAP@0.5..0.95 (%)': '0.361'}, " + \
+                "'Small (area < 32^2)': {'AP@.50 (%)': '0.288'}, 'Medium (32^2 < area < 96^2)': {'AP@.75 (%)': '0.369'}}"
+    ))
+
+def test_coco_area_param(coco_test_visiondata, trained_yolov5_object_detection):
+    # Arrange
+    pred_formatter = DetectionPredictionFormatter(yolo_prediction_formatter)
+    check = MeanAveragePrecisionReport(area_range=(40**2, 100**2))
+
+    # Act
+    result = check.run(coco_test_visiondata,
+                       trained_yolov5_object_detection, prediction_formatter=pred_formatter)
+
+    # Assert
+    df = result.value
+    assert_that(df, has_length(4))
+
+    assert_that(df.loc['All', 'mAP@0.5..0.95 (%)'], close_to(0.361, 0.001))
+    assert_that(df.loc['All', 'AP@.50 (%)'], close_to(0.501, 0.001))
+    assert_that(df.loc['All', 'AP@.75 (%)'], close_to(0.376, 0.001))
+
+    assert_that(df.loc['Small (area < 40^2)', 'mAP@0.5..0.95 (%)'], close_to(0.166, 0.001))
+    assert_that(df.loc['Small (area < 40^2)', 'AP@.50 (%)'], close_to(0.26, 0.001))
+    assert_that(df.loc['Small (area < 40^2)', 'AP@.75 (%)'], close_to(0.173, 0.001))
+
+    assert_that(df.loc['Medium (40^2 < area < 100^2)', 'mAP@0.5..0.95 (%)'], close_to(0.392, 0.001))
+    assert_that(df.loc['Medium (40^2 < area < 100^2)', 'AP@.50 (%)'], close_to(0.617, 0.001))
+    assert_that(df.loc['Medium (40^2 < area < 100^2)', 'AP@.75 (%)'], close_to(0.36, 0.001))
+
+    assert_that(df.loc['Large (area < 100^2)', 'mAP@0.5..0.95 (%)'], close_to(0.473, 0.001))
+    assert_that(df.loc['Large (area < 100^2)', 'AP@.50 (%)'], close_to(0.574, 0.001))
+    assert_that(df.loc['Large (area < 100^2)', 'AP@.75 (%)'], close_to(0.531, 0.001))
diff --git a/tests/vision/utils_tests/metrics_test.py b/tests/vision/utils_tests/metrics_test.py
--- a/tests/vision/utils_tests/metrics_test.py
+++ b/tests/vision/utils_tests/metrics_test.py
@@ -39,21 +39,23 @@ def test_equal_pycocotools(coco_test_visiondata: VisionData, trained_yolov5_obje
         metric.update((prediction, label))
     res = metric.compute()[0]
 
-    assert_that(metric.get_val_at(res['precision'], area='all', max_dets=100), close_to(0.361, 0.001))
-    assert_that(metric.get_val_at(res['precision'], iou=0.5, area='all', max_dets=100), close_to(0.502, 0.001))
-    assert_that(metric.get_val_at(res['precision'], iou=0.75, area='all', max_dets=100), close_to(0.376, 0.001))
-    assert_that(metric.get_val_at(res['precision'], area='small', max_dets=100), close_to(0.189, 0.001))
-    assert_that(metric.get_val_at(res['precision'], area='medium', max_dets=100), close_to(0.367, 0.001))
-    assert_that(metric.get_val_at(res['precision'], area='large', max_dets=100), close_to(0.476, 0.001))
-
-    assert_that(metric.get_val_at(res['recall'], area='all', max_dets=1), close_to(0.3, 0.001))
-    assert_that(metric.get_val_at(res['recall'], area='all', max_dets=10), close_to(0.379, 0.001))
-    assert_that(metric.get_val_at(res['recall'], area='all', max_dets=100), close_to(0.388, 0.001))
-    assert_that(metric.get_val_at(res['recall'], area='small', max_dets=100), close_to(0.194, 0.001))
-    assert_that(metric.get_val_at(res['recall'], area='medium', max_dets=100), close_to(0.403, 0.001))
-    assert_that(metric.get_val_at(res['recall'], area='large', max_dets=100), close_to(0.488, 0.001))
+    assert_that(metric.get_classes_scores_at(res['precision'], area='all', max_dets=100), close_to(0.361, 0.001))
+    assert_that(metric.get_classes_scores_at(res['precision'], iou=0.5, area='all', max_dets=100),
+                close_to(0.502, 0.001))
+    assert_that(metric.get_classes_scores_at(res['precision'], iou=0.75, area='all', max_dets=100),
+                close_to(0.376, 0.001))
+    assert_that(metric.get_classes_scores_at(res['precision'], area='small', max_dets=100), close_to(0.189, 0.001))
+    assert_that(metric.get_classes_scores_at(res['precision'], area='medium', max_dets=100), close_to(0.367, 0.001))
+    assert_that(metric.get_classes_scores_at(res['precision'], area='large', max_dets=100), close_to(0.476, 0.001))
+
+    assert_that(metric.get_classes_scores_at(res['recall'], area='all', max_dets=1), close_to(0.3, 0.001))
+    assert_that(metric.get_classes_scores_at(res['recall'], area='all', max_dets=10), close_to(0.379, 0.001))
+    assert_that(metric.get_classes_scores_at(res['recall'], area='all', max_dets=100), close_to(0.388, 0.001))
+    assert_that(metric.get_classes_scores_at(res['recall'], area='small', max_dets=100), close_to(0.194, 0.001))
+    assert_that(metric.get_classes_scores_at(res['recall'], area='medium', max_dets=100), close_to(0.403, 0.001))
+    assert_that(metric.get_classes_scores_at(res['recall'], area='large', max_dets=100), close_to(0.488, 0.001))
 
     # unrelated to coco but needed to check another param
-    assert_that(metric.get_val_at(res['recall'], area='large', max_dets=100, get_mean_val=False,
+    assert_that(metric.get_classes_scores_at(res['recall'], area='large', max_dets=100, get_mean_val=False,
                 zeroed_negative=False), has_items([-1]))
-    assert_that(metric.get_val_at(res['recall'], get_mean_val=False, zeroed_negative=False), has_items([-1]))
+    assert_that(metric.get_classes_scores_at(res['recall'], get_mean_val=False, zeroed_negative=False), has_items([-1]))