Repository: deepchecks/deepchecks

Issue:
[FEAT][CV] Simple model comparison needs to be evaluated on a global metric
**Is your feature request related to a problem? Please describe.**
Currently the simple model always gets perfect recall, which results in the condition always triggering.

**Describe the solution you'd like**
Like in the tabular case, the metric used to compare to the simple model should be a global metric (not per class), such as accuracy.



Discussion:

Resolving patch:
diff --git a/deepchecks/vision/checks/performance/simple_model_comparison.py b/deepchecks/vision/checks/performance/simple_model_comparison.py
--- a/deepchecks/vision/checks/performance/simple_model_comparison.py
+++ b/deepchecks/vision/checks/performance/simple_model_comparison.py
@@ -15,7 +15,7 @@
 import pandas as pd
 import plotly.express as px
 import torch
-from ignite.metrics import Metric
+from ignite.metrics import Fbeta
 
 from deepchecks.core import CheckResult, DatasetKind, ConditionResult, ConditionCategory
 from deepchecks.core.errors import DeepchecksValueError
@@ -83,7 +83,7 @@ class SimpleModelComparison(TrainTestCheck):
 
     def __init__(self,
                  strategy: str = 'most_frequent',
-                 alternative_metrics: Dict[str, Metric] = None,
+                 alternative_metrics=None,
                  n_to_show: int = 20,
                  show_only: str = 'largest',
                  metric_to_show_by: str = None,
@@ -119,8 +119,12 @@ def initialize_run(self, context: Context):
         """Initialize the metrics for the check, and validate task type is relevant."""
         context.assert_task_type(TaskType.CLASSIFICATION)
 
-        self._test_metrics = get_scorers_list(context.train, self.alternative_metrics)
-        self._perfect_metrics = get_scorers_list(context.train, self.alternative_metrics)
+        if self.alternative_metrics is None:
+            self._test_metrics = {'F1': Fbeta(beta=1, average=False)}
+            self._perfect_metrics = {'F1': Fbeta(beta=1, average=False)}
+        else:
+            self._test_metrics = get_scorers_list(context.train, self.alternative_metrics)
+            self._perfect_metrics = get_scorers_list(context.train, self.alternative_metrics)
 
     def update(self, context: Context, batch: Batch, dataset_kind: DatasetKind):
         """Update the metrics for the check."""
@@ -232,7 +236,10 @@ def _generate_simple_model_metrics(self, train, test):
                 dummy_predictions.append(dummy_predictor())
 
         # Get scorers
-        metrics = get_scorers_list(train, self.alternative_metrics)
+        if self.alternative_metrics is None:
+            metrics = {'F1': Fbeta(beta=1, average=False)}
+        else:
+            metrics = get_scorers_list(train, self.alternative_metrics)
         for _, metric in metrics.items():
             metric.update((torch.stack(dummy_predictions), torch.LongTensor(labels)))
         return metrics


Test patch:
diff --git a/tests/vision/checks/performance/cv_simple_model_comparison_test.py b/tests/vision/checks/performance/cv_simple_model_comparison_test.py
--- a/tests/vision/checks/performance/cv_simple_model_comparison_test.py
+++ b/tests/vision/checks/performance/cv_simple_model_comparison_test.py
@@ -25,8 +25,8 @@ def test_mnist_prior_strategy(mnist_dataset_train, mnist_dataset_test, mock_trai
     first_row = result.value.loc[result.value['Model'] != 'Perfect Model'].sort_values(by='Number of samples',
                                                                                        ascending=False).iloc[0]
     # Assert
-    assert_that(len(result.value), equal_to(12))
-    assert_that(first_row['Value'], close_to(0.994, 0.05))
+    assert_that(len(result.value), equal_to(6))
+    assert_that(first_row['Value'], close_to(0.203, 0.05))
     assert_that(first_row['Number of samples'], equal_to(1135))
     assert_that(first_row['Class'], equal_to(1))
 
@@ -49,8 +49,8 @@ def test_mnist_most_frequent(mnist_dataset_train, mnist_dataset_test, mock_train
     first_row = result.value.loc[result.value['Model'] != 'Perfect Model'].sort_values(by='Number of samples',
                                                                                        ascending=False).iloc[0]
     # Assert
-    assert_that(len(result.value), equal_to(12))
-    assert_that(first_row['Value'], close_to(0.994, 0.05))
+    assert_that(len(result.value), equal_to(6))
+    assert_that(first_row['Value'], close_to(0.203, 0.05))
     assert_that(first_row['Number of samples'], equal_to(1135))
     assert_that(first_row['Class'], equal_to(1))
 
@@ -62,7 +62,7 @@ def test_mnist_uniform(mnist_dataset_train, mnist_dataset_test, mock_trained_mni
     result = check.run(mnist_dataset_train, mnist_dataset_test, mock_trained_mnist,
                        device=device)
     # Assert
-    assert_that(len(result.value), equal_to(12))
+    assert_that(len(result.value), equal_to(6))
 
 
 def test_mnist_stratified(mnist_dataset_train, mnist_dataset_test, mock_trained_mnist, device):
@@ -72,21 +72,21 @@ def test_mnist_stratified(mnist_dataset_train, mnist_dataset_test, mock_trained_
     result = check.run(mnist_dataset_train, mnist_dataset_test, mock_trained_mnist,
                        device=device)
     # Assert
-    assert_that(len(result.value), equal_to(12))
+    assert_that(len(result.value), equal_to(6))
 
 
 def test_condition_failed_for_multiclass(mnist_dataset_train, mnist_dataset_test, mock_trained_mnist, device):
     train_ds, test_ds, clf = mnist_dataset_train, mnist_dataset_test, mock_trained_mnist
     # Arrange
-    check = SimpleModelComparison().add_condition_gain_not_less_than(0.8)
+    check = SimpleModelComparison().add_condition_gain_not_less_than(0.973)
     # Act X
     result = check.run(train_ds, test_ds, clf)
     # Assert
     assert_that(result.conditions_results, has_items(
         equal_condition_result(
             is_pass=False,
-            name='Model performance gain over simple model is not less than 80%',
-            details='Found metrics with gain below threshold: {\'Recall\': {1: \'-5000%\'}}')
+            name='Model performance gain over simple model is not less than 97.3%',
+            details='Found metrics with gain below threshold: {\'F1\': {9: \'97.27%\'}}')
 
     ))
 
@@ -117,6 +117,6 @@ def test_condition_pass_for_multiclass_avg_with_classes(mnist_dataset_train, mni
         equal_condition_result(
             is_pass=False,
             name='Model performance gain over simple model is not less than 100% for classes [0]',
-            details='Found metrics with gain below threshold: {\'Recall\': \'99.39%\', \'Precision\': \'97.89%\'}'
+            details='Found metrics with gain below threshold: {\'F1\': \'98.63%\'}'
         )
     ))