Repository: deepchecks/deepchecks

Issue:
[FEAT][CV] Average Recall metric
**Describe the solution you'd like**
I think the average recall check for object detection use-cases is really informative. 
Average recall means testing the recall at different IoU thresholds and showing on the graph the recall value per each threshold.

This check should return the followings:
- Average recall graph (described above)
- mAR value
- AR for different object dimensions:
  - for small objects (IoU range 0<x<32^2]
  - for medium objects (IoU range 32^2<x<96^2)
  - for large objects (IoU range x>96^2)
- AR for different number of detections per image:
  - For max 1 detection per image
  - For max 10 detections per image
  - For max 100 detections per image

Please see the COCO evaluation metrics for more information


Discussion:

Resolving patch:
diff --git a/deepchecks/vision/checks/performance/__init__.py b/deepchecks/vision/checks/performance/__init__.py
--- a/deepchecks/vision/checks/performance/__init__.py
+++ b/deepchecks/vision/checks/performance/__init__.py
@@ -11,10 +11,12 @@
 """Module containing the performance check in the vision package."""
 from .class_performance import ClassPerformance
 from .mean_average_precision_report import MeanAveragePrecisionReport
+from .mean_average_recall_report import MeanAverageRecallReport
 from .robustness_report import RobustnessReport
 
 __all__ = [
     "ClassPerformance",
     "MeanAveragePrecisionReport",
+    "MeanAverageRecallReport",
     "RobustnessReport",
 ]
diff --git a/deepchecks/vision/checks/performance/mean_average_recall_report.py b/deepchecks/vision/checks/performance/mean_average_recall_report.py
new file mode 100644
--- /dev/null
+++ b/deepchecks/vision/checks/performance/mean_average_recall_report.py
@@ -0,0 +1,98 @@
+# ----------------------------------------------------------------------------
+# Copyright (C) 2021-2022 Deepchecks (https://www.deepchecks.com)
+#
+# This file is part of Deepchecks.
+# Deepchecks is distributed under the terms of the GNU Affero General
+# Public License (version 3 or later).
+# You should have received a copy of the GNU Affero General Public License
+# along with Deepchecks.  If not, see <http://www.gnu.org/licenses/>.
+# ----------------------------------------------------------------------------
+#
+"""Module containing mean average recall report check."""
+from collections import defaultdict
+import math
+from typing import TypeVar, Tuple, Any
+
+import pandas as pd
+
+from deepchecks.core import CheckResult, ConditionResult, DatasetKind
+from deepchecks.utils.strings import format_number
+from deepchecks.vision import SingleDatasetCheck, Context
+from deepchecks.vision.dataset import TaskType
+from deepchecks.vision.metrics_utils.detection_precision_recall import AveragePrecision
+
+__all__ = ['MeanAverageRecallReport']
+
+MPR = TypeVar('MPR', bound='MeanAverageRecallReport')
+
+
+class MeanAverageRecallReport(SingleDatasetCheck):
+    """Summarize mean average recall metrics on a dataset and model per detections and area range.
+
+    Parameters
+    ----------
+    area_range: tuple, default: (32**2, 96**2)
+        Slices for small/medium/large buckets.
+    """
+
+    def __init__(self, area_range: Tuple = (32**2, 96**2)):
+        super().__init__()
+        self._ap_metric = AveragePrecision(return_option=None, area_range=area_range)
+
+    def initialize_run(self, context: Context, dataset_kind: DatasetKind = None):
+        """Initialize run by asserting task type and initializing metric."""
+        context.assert_task_type(TaskType.OBJECT_DETECTION)
+
+    def update(self, context: Context, batch: Any, dataset_kind: DatasetKind):
+        """Update the metrics by passing the batch to ignite metric update method."""
+        dataset = context.get_data_by_kind(dataset_kind)
+        images = batch[0]
+        label = dataset.label_transformer(batch[1])
+        prediction = context.prediction_formatter(context.infer(images))
+        self._ap_metric.update((prediction, label))
+
+    def compute(self, context: Context, dataset_kind: DatasetKind) -> CheckResult:
+        """Compute the metric result using the ignite metrics compute method and create display."""
+        small_area = int(math.sqrt(self._ap_metric.area_range[0]))
+        large_area = int(math.sqrt(self._ap_metric.area_range[1]))
+        res = self._ap_metric.compute()[0]['recall']
+        rows = []
+        for title, area_name in zip(['All',
+                                     f'Small (area < {small_area}^2)',
+                                     f'Medium ({small_area}^2 < area < {large_area}^2)',
+                                     f'Large (area < {large_area}^2)'],
+                                    ['all', 'small', 'medium', 'large']):
+            area_scores = [title]
+            area_scores.append(self._ap_metric.get_classes_scores_at(res, area=area_name, max_dets=1))
+            area_scores.append(self._ap_metric.get_classes_scores_at(res, area=area_name, max_dets=10))
+            area_scores.append(self._ap_metric.get_classes_scores_at(res, area=area_name, max_dets=100))
+
+            rows.append(area_scores)
+
+        results = pd.DataFrame(columns=['Area size', 'AR@1 (%)', 'AR@10 (%)', 'AR@100 (%)'])
+        for i in range(len(rows)):
+            results.loc[i] = rows[i]
+        results = results.set_index('Area size')
+
+        return CheckResult(value=results, display=[results])
+
+    def add_condition_test_average_recall_not_less_than(self: MPR, min_score: float) -> MPR:
+        """Add condition - AR score is not less than given score.
+
+        Parameters
+        ----------
+        min_score : float
+            Minimum score to pass the check.
+        """
+        def condition(df: pd.DataFrame):
+            not_passed = defaultdict(dict)
+            for index, column in zip(df.index, df.columns):
+                if df.loc[index, column] < min_score:
+                    not_passed[index][column] = format_number(df.loc[index, column], 3)
+            if len(not_passed):
+                details = f'Found scores below threshold:\n' \
+                          f'{dict(not_passed)}'
+                return ConditionResult(False, details)
+            return ConditionResult(True)
+
+        return self.add_condition(f'Scores are not less than {min_score}', condition)


Test patch:
diff --git a/tests/vision/checks/performance/mean_average_recall_report_test.py b/tests/vision/checks/performance/mean_average_recall_report_test.py
new file mode 100644
--- /dev/null
+++ b/tests/vision/checks/performance/mean_average_recall_report_test.py
@@ -0,0 +1,99 @@
+# ----------------------------------------------------------------------------
+# Copyright (C) 2021 Deepchecks (https://www.deepchecks.com)
+#
+# This file is part of Deepchecks.
+# Deepchecks is distributed under the terms of the GNU Affero General
+# Public License (version 3 or later).
+# You should have received a copy of the GNU Affero General Public License
+# along with Deepchecks.  If not, see <http://www.gnu.org/licenses/>.
+# ----------------------------------------------------------------------------
+#
+import re
+from hamcrest import assert_that, close_to, has_length, calling, raises
+
+from tests.checks.utils import equal_condition_result
+from deepchecks.core.errors import ModelValidationError
+from deepchecks.vision.checks.performance import MeanAverageRecallReport
+from deepchecks.vision.datasets.detection.coco import yolo_prediction_formatter
+from deepchecks.vision.utils import DetectionPredictionFormatter
+
+def test_mnist_error(mnist_dataset_test, trained_mnist):
+    # Arrange
+    check = MeanAverageRecallReport()
+    # Act
+    assert_that(
+        calling(check.run).with_args(mnist_dataset_test, trained_mnist),
+            raises(ModelValidationError, r'Check is irrelevant for task of type TaskType.CLASSIFICATION')
+    )
+
+
+def test_coco(coco_test_visiondata, trained_yolov5_object_detection):
+    # Arrange
+    pred_formatter = DetectionPredictionFormatter(yolo_prediction_formatter)
+    check = MeanAverageRecallReport() \
+            .add_condition_test_average_recall_not_less_than(0.1) \
+            .add_condition_test_average_recall_not_less_than(0.4)
+
+    # Act
+    result = check.run(coco_test_visiondata,
+                       trained_yolov5_object_detection, prediction_formatter=pred_formatter)
+
+    # Assert
+    df = result.value
+    assert_that(df, has_length(4))
+
+    assert_that(df.loc['All', 'AR@1 (%)'], close_to(0.299, 0.001))
+    assert_that(df.loc['All', 'AR@10 (%)'], close_to(0.379, 0.001))
+    assert_that(df.loc['All', 'AR@100 (%)'], close_to(0.387, 0.001))
+
+    assert_that(df.loc['Small (area < 32^2)', 'AR@1 (%)'], close_to(0.091, 0.001))
+    assert_that(df.loc['Small (area < 32^2)', 'AR@10 (%)'], close_to(0.167, 0.001))
+    assert_that(df.loc['Small (area < 32^2)', 'AR@100 (%)'], close_to(0.194, 0.001))
+
+    assert_that(df.loc['Medium (32^2 < area < 96^2)', 'AR@1 (%)'], close_to(0.315, 0.001))
+    assert_that(df.loc['Medium (32^2 < area < 96^2)', 'AR@10 (%)'], close_to(0.392, 0.001))
+    assert_that(df.loc['Medium (32^2 < area < 96^2)', 'AR@100 (%)'], close_to(0.403, 0.001))
+
+    assert_that(df.loc['Large (area < 96^2)', 'AR@1 (%)'], close_to(0.426, 0.001))
+    assert_that(df.loc['Large (area < 96^2)', 'AR@10 (%)'], close_to(0.483, 0.001))
+    assert_that(df.loc['Large (area < 96^2)', 'AR@100 (%)'], close_to(0.488, 0.001))
+
+    assert_that(result.conditions_results[0], equal_condition_result(
+        is_pass=True,
+        name='Scores are not less than 0.1'
+    ))
+
+    assert_that(result.conditions_results[1], equal_condition_result(
+        is_pass=False,
+        name='Scores are not less than 0.4',
+        details="Found scores below threshold:\n{'All': {'AR@1 (%)': '0.3'}, 'Small (area < 32^2)': {'AR@10 (%)': '0.168'}}"
+    ))
+
+def test_coco_area_param(coco_test_visiondata, trained_yolov5_object_detection):
+    # Arrange
+    pred_formatter = DetectionPredictionFormatter(yolo_prediction_formatter)
+    check = MeanAverageRecallReport(area_range=(40**2, 100**2))
+
+    # Act
+    result = check.run(coco_test_visiondata,
+                       trained_yolov5_object_detection, prediction_formatter=pred_formatter)
+
+    # Assert
+    df = result.value
+    assert_that(df, has_length(4))
+
+    assert_that(df.loc['All', 'AR@1 (%)'], close_to(0.299, 0.001))
+    assert_that(df.loc['All', 'AR@10 (%)'], close_to(0.379, 0.001))
+    assert_that(df.loc['All', 'AR@100 (%)'], close_to(0.387, 0.001))
+
+    assert_that(df.loc['Small (area < 40^2)', 'AR@1 (%)'], close_to(0.089, 0.001))
+    assert_that(df.loc['Small (area < 40^2)', 'AR@10 (%)'], close_to(0.167, 0.001))
+    assert_that(df.loc['Small (area < 40^2)', 'AR@100 (%)'], close_to(0.18, 0.001))
+
+    assert_that(df.loc['Medium (40^2 < area < 100^2)', 'AR@1 (%)'], close_to(0.309, 0.001))
+    assert_that(df.loc['Medium (40^2 < area < 100^2)', 'AR@10 (%)'], close_to(0.408, 0.001))
+    assert_that(df.loc['Medium (40^2 < area < 100^2)', 'AR@100 (%)'], close_to(0.431, 0.001))
+
+    assert_that(df.loc['Large (area < 100^2)', 'AR@1 (%)'], close_to(0.424, 0.001))
+    assert_that(df.loc['Large (area < 100^2)', 'AR@10 (%)'], close_to(0.48, 0.001))
+    assert_that(df.loc['Large (area < 100^2)', 'AR@100 (%)'], close_to(0.484, 0.001))