Repository: deepchecks/deepchecks

Issue:
[FEAT][CV] Treat multiple labels in performance report
Currently, metrics are calculated per class and displayed per class. In cases when there are many classes (e.g. > 30), we want an alternative way of inspecting metrics only for specific classes.

Current idea:
1. Have a per_class flag, default=True, to enable also displaying not per class
2. Have a n_to_show=20 arg, to control how many classes to show.
3. show_only = 'largest' arg, which controls the method by which the classes to show are selected. Can be either a list (assumed to be list of classes to show), or "largest", "smallest", "random", "best", "worst". Other ideas are welcome.


Discussion:
@noamzbr what will happen on ClassPerformance(per_class=False)?
Class performance shouldn't have the "per_class" param, only the others
(or alternatively it should run as performance_report when per_class is true)

Resolving patch:
diff --git a/deepchecks/vision/checks/performance/class_performance.py b/deepchecks/vision/checks/performance/class_performance.py
--- a/deepchecks/vision/checks/performance/class_performance.py
+++ b/deepchecks/vision/checks/performance/class_performance.py
@@ -20,8 +20,8 @@
 from deepchecks.utils.strings import format_percent, format_number
 from deepchecks.vision import TrainTestCheck, Context
 from deepchecks.vision.dataset import TaskType
-from deepchecks.vision.metrics_utils.metrics import get_scorers_list, metric_results_to_df
-
+from deepchecks.vision.metrics_utils.metrics import get_scorers_list, metric_results_to_df, \
+    get_default_classification_scorers, get_default_object_detection_scorers
 
 __all__ = ['ClassPerformance']
 
@@ -35,19 +35,60 @@ class ClassPerformance(TrainTestCheck):
     ----------
     alternative_metrics : List[Metric], default: None
         A list of ignite.Metric objects whose score should be used. If None are given, use the default metrics.
+    n_to_show : int, default: 20
+        Number of classes to show in the report. If None, show all classes.
+    show_only : str, default: 'largest'
+        Specify which classes to show in the report. Can be one of the following:
+        - 'largest': Show the largest classes.
+        - 'smallest': Show the smallest classes.
+        - 'random': Show random classes.
+        - 'best': Show the classes with the highest score.
+        - 'worst': Show the classes with the lowest score.
+    metric_to_show_by : str, default: None
+        Specify the metric to sort the results by. Relevant only when show_only is 'best' or 'worst'.
+        If None, sorting by the first metric in the default metrics list.
+    class_list_to_show: List[int], default: None
+        Specify the list of classes to show in the report. If specified, n_to_show, show_only and metric_to_show_by
+        are ignored.
     """
 
     def __init__(self,
-                 alternative_metrics: List[Metric] = None
-                 ):
+                 alternative_metrics: List[Metric] = None,
+                 n_to_show: int = 20,
+                 show_only: str = 'largest',
+                 metric_to_show_by: str = None,
+                 class_list_to_show: List[int] = None):
         super().__init__()
         self.alternative_metrics = alternative_metrics
+        self.n_to_show = n_to_show
+        self.class_list_to_show = class_list_to_show
+
+        if self.class_list_to_show is None:
+            if show_only not in ['largest', 'smallest', 'random', 'best', 'worst']:
+                raise DeepchecksValueError(f'Invalid value for show_only: {show_only}. Should be one of: '
+                                           f'["largest", "smallest", "random", "best", "worst"]')
+
+            self.show_only = show_only
+            if alternative_metrics is not None and show_only in ['best', 'worst'] and metric_to_show_by is None:
+                raise DeepchecksValueError('When alternative_metrics are provided and show_only is one of: '
+                                           '["best", "worst"], metric_to_show_by must be specified.')
+
+        self.metric_to_show_by = metric_to_show_by
+
         self._state = {}
 
     def initialize_run(self, context: Context):
         """Initialize run by creating the _state member with metrics for train and test."""
         context.assert_task_type(TaskType.CLASSIFICATION, TaskType.OBJECT_DETECTION)
 
+        if not self.metric_to_show_by:
+            if context.train.task_type == TaskType.CLASSIFICATION:
+                self.metric_to_show_by = list(get_default_classification_scorers().keys())[0]
+            elif context.train.task_type == TaskType.OBJECT_DETECTION:
+                self.metric_to_show_by = list(get_default_object_detection_scorers().keys())[0]
+            else:
+                raise DeepchecksValueError(f'Invalid task type: {context.train.task_type}')
+
         self._state = {DatasetKind.TRAIN: {}, DatasetKind.TEST: {}}
         self._state[DatasetKind.TRAIN]['scorers'] = get_scorers_list(context.train, self.alternative_metrics)
         self._state[DatasetKind.TEST]['scorers'] = get_scorers_list(context.train, self.alternative_metrics)
@@ -75,6 +116,14 @@ def compute(self, context: Context) -> CheckResult:
 
         results_df = pd.concat(results)
 
+        if self.class_list_to_show is not None:
+            results_df = results_df.loc[results_df['Class'].isin(self.class_list_to_show)]
+        elif self.n_to_show is not None:
+            classes_to_show = self._filter_classes(results_df)
+            results_df = results_df.loc[results_df['Class'].isin(classes_to_show)]
+
+        results_df = results_df.sort_values(by=['Dataset', 'Value'], ascending=False)
+
         fig = px.histogram(
             results_df,
             x='Class',
@@ -102,6 +151,25 @@ def compute(self, context: Context) -> CheckResult:
             display=fig
         )
 
+    def _filter_classes(self, metrics_df: pd.DataFrame) -> list:
+        # working only on the test set
+        tests_metrics_df = metrics_df[(metrics_df['Dataset'] == DatasetKind.TEST.value) &
+                                      (metrics_df['Metric'] == self.metric_to_show_by)]
+        if self.show_only == 'largest':
+            tests_metrics_df = tests_metrics_df.sort_values(by='Number of samples', ascending=False)
+        elif self.show_only == 'smallest':
+            tests_metrics_df = tests_metrics_df.sort_values(by='Number of samples', ascending=True)
+        elif self.show_only == 'random':
+            tests_metrics_df = tests_metrics_df.sample(frac=1)
+        elif self.show_only == 'best':
+            tests_metrics_df = tests_metrics_df.sort_values(by='Value', ascending=False)
+        elif self.show_only == 'worst':
+            tests_metrics_df = tests_metrics_df.sort_values(by='Value', ascending=True)
+        else:
+            raise ValueError(f'Unknown show_only value: {self.show_only}')
+
+        return tests_metrics_df.head(self.n_to_show)['Class'].to_list()
+
     def add_condition_test_performance_not_less_than(self: PR, min_score: float) -> PR:
         """Add condition - metric scores are not less than given score.
 


Test patch:
diff --git a/tests/vision/checks/performance/class_performance_test.py b/tests/vision/checks/performance/class_performance_test.py
new file mode 100644
--- /dev/null
+++ b/tests/vision/checks/performance/class_performance_test.py
@@ -0,0 +1,110 @@
+# ----------------------------------------------------------------------------
+# Copyright (C) 2021 Deepchecks (https://www.deepchecks.com)
+#
+# This file is part of Deepchecks.
+# Deepchecks is distributed under the terms of the GNU Affero General
+# Public License (version 3 or later).
+# You should have received a copy of the GNU Affero General Public License
+# along with Deepchecks.  If not, see <http://www.gnu.org/licenses/>.
+# ----------------------------------------------------------------------------
+#
+from deepchecks.vision.checks.performance.class_performance import ClassPerformance
+from deepchecks.vision.datasets.detection.coco import yolo_prediction_formatter
+from deepchecks.vision.utils import DetectionPredictionFormatter
+from deepchecks.vision.utils.classification_formatters import ClassificationPredictionFormatter
+
+import torch.nn as nn
+
+from hamcrest import assert_that, has_entries, close_to, equal_to, is_in
+from tests.vision.vision_conftest import *
+
+
+def test_mnist_largest(mnist_dataset_train, mnist_dataset_test, trained_mnist):
+    # Arrange
+
+    check = ClassPerformance(n_to_show=2, show_only='largest')
+    # Act
+    result = check.run(mnist_dataset_train, mnist_dataset_test, trained_mnist,
+                       prediction_formatter=ClassificationPredictionFormatter(nn.Softmax(dim=1)))
+    first_row = result.value.sort_values(by='Number of samples', ascending=False).iloc[0]
+    # Assert
+    assert_that(len(result.value), equal_to(8))
+    assert_that(first_row['Value'], close_to(0.991532, 0.05))
+    assert_that(first_row['Number of samples'], equal_to(6742))
+    assert_that(first_row['Class'], equal_to(1))
+
+
+def test_mnist_smallest(mnist_dataset_train, mnist_dataset_test, trained_mnist):
+    # Arrange
+
+    check = ClassPerformance(n_to_show=2, show_only='smallest')
+    # Act
+    result = check.run(mnist_dataset_train, mnist_dataset_test, trained_mnist,
+                       prediction_formatter=ClassificationPredictionFormatter(nn.Softmax(dim=1)))
+    first_row = result.value.sort_values(by='Number of samples', ascending=True).iloc[0]
+
+    # Assert
+    assert_that(len(result.value), equal_to(8))
+    assert_that(first_row['Value'], close_to(0.988739, 0.05))
+    assert_that(first_row['Number of samples'], equal_to(892))
+    assert_that(first_row['Class'], equal_to(5))
+
+
+def test_mnist_worst(mnist_dataset_train, mnist_dataset_test, trained_mnist):
+    # Arrange
+
+    check = ClassPerformance(n_to_show=2, show_only='worst')
+    # Act
+    result = check.run(mnist_dataset_train, mnist_dataset_test, trained_mnist,
+                       prediction_formatter=ClassificationPredictionFormatter(nn.Softmax(dim=1)))
+    first_row = result.value.loc[result.value['Metric'] == 'Precision'].sort_values(by='Value', ascending=True).iloc[0]
+
+    # Assert
+    assert_that(len(result.value), equal_to(8))
+    assert_that(first_row['Value'], close_to(0.977713, 0.05))
+
+
+def test_mnist_best(mnist_dataset_train, mnist_dataset_test, trained_mnist):
+    # Arrange
+
+    check = ClassPerformance(n_to_show=2, show_only='best')
+    # Act
+    result = check.run(mnist_dataset_train, mnist_dataset_test, trained_mnist,
+                       prediction_formatter=ClassificationPredictionFormatter(nn.Softmax(dim=1)))
+    first_row = result.value.loc[result.value['Metric'] == 'Precision'].sort_values(by='Value', ascending=False).iloc[0]
+
+    # Assert
+    assert_that(len(result.value), equal_to(8))
+    assert_that(first_row['Value'], close_to(0.990854, 0.05))
+
+
+def test_coco_best(coco_train_visiondata, coco_test_visiondata, trained_yolov5_object_detection):
+    # Arrange
+    pred_formatter = DetectionPredictionFormatter(yolo_prediction_formatter)
+    check = ClassPerformance(n_to_show=2, show_only='best')
+    # Act
+    result = check.run(coco_train_visiondata, coco_test_visiondata,
+                       trained_yolov5_object_detection, prediction_formatter=pred_formatter)
+    first_row = result.value.loc[result.value['Metric'] == 'mAP'].sort_values(by='Value', ascending=False).iloc[0]
+
+    # Assert
+    assert_that(len(result.value), equal_to(4))
+    assert_that(first_row['Value'], close_to(0.990854, 0.05))
+    assert_that(first_row['Number of samples'], equal_to(1))
+    assert_that(first_row['Class'], is_in([28, 40]))
+
+
+def test_class_list(mnist_dataset_train, mnist_dataset_test, trained_mnist):
+    # Arrange
+
+    check = ClassPerformance(class_list_to_show=[1])
+    # Act
+    result = check.run(mnist_dataset_train, mnist_dataset_test, trained_mnist,
+                       prediction_formatter=ClassificationPredictionFormatter(nn.Softmax(dim=1)))
+
+    # Assert
+    assert_that(len(result.value), equal_to(4))
+    assert_that(result.value['Class'].iloc[0], equal_to(1))
+    assert_that(result.value['Class'].iloc[1], equal_to(1))
+    assert_that(result.value['Class'].iloc[2], equal_to(1))
+    assert_that(result.value['Class'].iloc[3], equal_to(1))
\ No newline at end of file