Repository: deepchecks/deepchecks

Issue:
[BUG] [CV] No condition for simple model check
No condition for simple model check


Discussion:

Resolving patch:
diff --git a/deepchecks/vision/checks/performance/simple_model_comparison.py b/deepchecks/vision/checks/performance/simple_model_comparison.py
--- a/deepchecks/vision/checks/performance/simple_model_comparison.py
+++ b/deepchecks/vision/checks/performance/simple_model_comparison.py
@@ -17,8 +17,10 @@
 import torch
 from ignite.metrics import Metric
 
-from deepchecks.core import CheckResult, DatasetKind
+from deepchecks.core import CheckResult, DatasetKind, ConditionResult, ConditionCategory
 from deepchecks.core.errors import DeepchecksValueError
+from deepchecks.utils.metrics import get_gain
+from deepchecks.utils.strings import format_percent
 from deepchecks.vision import Context, TrainTestCheck, Batch
 from deepchecks.vision.vision_data import TaskType
 from deepchecks.vision.metrics_utils import get_scorers_list, metric_results_to_df
@@ -110,27 +112,36 @@ def __init__(self,
 
         self.metric_to_show_by = metric_to_show_by
         self._test_metrics = None
+        self._perfect_metrics = None
 
     def initialize_run(self, context: Context):
         """Initialize the metrics for the check, and validate task type is relevant."""
         context.assert_task_type(TaskType.CLASSIFICATION)
 
         self._test_metrics = get_scorers_list(context.train, self.alternative_metrics)
+        self._perfect_metrics = get_scorers_list(context.train, self.alternative_metrics)
 
     def update(self, context: Context, batch: Batch, dataset_kind: DatasetKind):
         """Update the metrics for the check."""
-        if dataset_kind == DatasetKind.TEST:
+        if dataset_kind == DatasetKind.TEST and context.train.task_type == TaskType.CLASSIFICATION:
             label = batch.labels
             prediction = batch.predictions
             for _, metric in self._test_metrics.items():
                 metric.update((prediction, label))
 
+            # calculating perfect scores
+            n_of_classes = batch.predictions.to('cpu').shape[1]
+            perfect_predictions = np.eye(n_of_classes)[label.to('cpu').numpy()]
+            for _, metric in self._perfect_metrics.items():
+                metric.update((torch.Tensor(perfect_predictions).to(context.device), label))
+
     def compute(self, context: Context) -> CheckResult:
         """Compute the metrics for the check."""
         results = []
 
         metrics_to_eval = {
             'Given Model': self._test_metrics,
+            'Perfect Model': self._perfect_metrics,
             'Simple Model': self._generate_simple_model_metrics(context.train, context.test)
         }
         for name, metrics in metrics_to_eval.items():
@@ -151,7 +162,7 @@ def compute(self, context: Context) -> CheckResult:
         if self.class_list_to_show is not None:
             results_df = results_df.loc[results_df['Class'].isin(self.class_list_to_show)]
         elif self.n_to_show is not None:
-            classes_to_show = filter_classes_for_display(results_df,
+            classes_to_show = filter_classes_for_display(results_df.loc[results_df['Model'] != 'Perfect Model'],
                                                          self.metric_to_show_by,
                                                          self.n_to_show,
                                                          self.show_only,
@@ -162,7 +173,7 @@ def compute(self, context: Context) -> CheckResult:
         results_df = results_df.sort_values(by=['Model', 'Value'], ascending=False).reset_index(drop=True)
 
         fig = px.histogram(
-            results_df,
+            results_df.loc[results_df['Model'] != 'Perfect Model'],
             x='Class Name',
             y='Value',
             color='Model',
@@ -221,3 +232,126 @@ def _generate_simple_model_metrics(self, train, test):
         for _, metric in metrics.items():
             metric.update((torch.stack(dummy_predictions), torch.LongTensor(labels)))
         return metrics
+
+    def add_condition_gain_not_less_than(self,
+                                         min_allowed_gain: float = 0.1,
+                                         max_gain: float = 50,
+                                         classes: List[Hashable] = None,
+                                         average: bool = False):
+        """Add condition - require minimum allowed gain between the model and the simple model.
+
+        Parameters
+        ----------
+        min_allowed_gain : float , default: 0.1
+            Minimum allowed gain between the model and the simple model -
+            gain is: difference in performance / (perfect score - simple score)
+        max_gain : float , default: 50
+            the maximum value for the gain value, limits from both sides [-max_gain, max_gain]
+        classes : List[Hashable] , default: None
+            Used in classification models to limit condition only to given classes.
+        average : bool , default: False
+            Used in classification models to flag if to run condition on average of classes, or on
+            each class individually
+        """
+        name = f'Model performance gain over simple model is not less than {format_percent(min_allowed_gain)}'
+        if classes:
+            name = name + f' for classes {str(classes)}'
+        return self.add_condition(name,
+                                  calculate_condition_logic,
+                                  include_classes=classes,
+                                  min_allowed_gain=min_allowed_gain,
+                                  max_gain=max_gain,
+                                  average=average)
+
+
+def calculate_condition_logic(result, include_classes=None, average=False, max_gain=None,
+                              min_allowed_gain=0) -> ConditionResult:
+    scores = result.loc[result['Model'] == 'Given Model']
+    perfect_scores = result.loc[result['Model'] == 'Perfect Model']
+    simple_scores = result.loc[result['Model'] == 'Simple Model']
+
+    metrics = scores['Metric'].unique()
+
+    fails = {}
+    if not average:
+        for metric in metrics:
+            failed_classes = {}
+            for _, scores_row in scores.loc[scores['Metric'] == metric].iterrows():
+                if include_classes and scores_row['Class'] not in include_classes:
+                    continue
+                perfect = perfect_scores.loc[(perfect_scores['Metric'] == metric) &
+                                             (perfect_scores['Class'] == scores_row['Class'])]['Value'].values[0]
+                if scores_row['Value'] == perfect:
+                    continue
+
+                simple_score_value = simple_scores.loc[(simple_scores['Class'] == scores_row['Class']) &
+                                                       (simple_scores['Metric'] == metric)]['Value'].values[0]
+                gain = get_gain(simple_score_value,
+                                scores_row['Value'],
+                                perfect,
+                                max_gain)
+                if gain < min_allowed_gain:
+                    failed_classes[scores_row['Class']] = format_percent(gain)
+
+            if failed_classes:
+                fails[metric] = failed_classes
+    else:
+        scores = average_scores(scores, simple_scores, include_classes)
+        for metric, models_scores in scores.items():
+            metric_perfect_score = perfect_scores.loc[(perfect_scores['Metric'] == metric)]['Value'].values[0]
+            # If origin model is perfect, skip the gain calculation
+            if models_scores['Origin'] == metric_perfect_score:
+                continue
+            gain = get_gain(models_scores['Simple'],
+                            models_scores['Origin'],
+                            metric_perfect_score,
+                            max_gain)
+            if gain < min_allowed_gain:
+                fails[metric] = format_percent(gain)
+
+    if fails:
+        msg = f'Found metrics with gain below threshold: {fails}'
+        return ConditionResult(ConditionCategory.FAIL, msg)
+    else:
+        return ConditionResult(ConditionCategory.PASS)
+
+
+def average_scores(scores, simple_model_scores, include_classes):
+    """
+    Calculate the average of the scores for each metric for all classes.
+
+    Parameters
+    ----------
+    scores : pd.DataFrame
+        the scores for the given model
+    simple_model_scores : pd.DataFrame
+        the scores for the simple model
+    include_classes : List[Hashable]
+        the classes to include in the calculation
+
+    Returns
+    -------
+    Dictionary[str, Dictionary[str, float]]
+        the average scores for each metric. The keys are the metric names, and the values are a dictionary
+        with the keys being Origin and Simple and the values being the average score.
+    """
+    result = {}
+    metrics = scores['Metric'].unique()
+    for metric in metrics:
+        model_score = 0
+        simple_score = 0
+        total = 0
+        for _, row in scores.loc[scores['Metric'] == metric].iterrows():
+            if include_classes and row['Class'] not in include_classes:
+                continue
+            model_score += row['Value']
+            simple_score += simple_model_scores.loc[(simple_model_scores['Class'] == row['Class']) &
+                                                    (simple_model_scores['Metric'] == metric)]['Value'].values[0]
+            total += 1
+
+        result[metric] = {
+            'Origin': model_score / total,
+            'Simple': simple_score / total
+         }
+
+    return result


Test patch:
diff --git a/tests/vision/checks/performance/cv_simple_model_comparison_test.py b/tests/vision/checks/performance/cv_simple_model_comparison_test.py
--- a/tests/vision/checks/performance/cv_simple_model_comparison_test.py
+++ b/tests/vision/checks/performance/cv_simple_model_comparison_test.py
@@ -10,7 +10,9 @@
 #
 from deepchecks.core.errors import DeepchecksValueError
 from deepchecks.vision.checks.performance.simple_model_comparison import SimpleModelComparison
-from hamcrest import assert_that, close_to, equal_to, is_in, calling, raises
+from hamcrest import assert_that, close_to, equal_to, is_in, calling, raises, has_items
+
+from tests.checks.utils import equal_condition_result
 
 
 def test_mnist_prior_strategy(mnist_dataset_train, mnist_dataset_test, mock_trained_mnist, device):
@@ -20,9 +22,10 @@ def test_mnist_prior_strategy(mnist_dataset_train, mnist_dataset_test, mock_trai
     # Act
     result = check.run(mnist_dataset_train, mnist_dataset_test, mock_trained_mnist,
                        device=device)
-    first_row = result.value.sort_values(by='Number of samples', ascending=False).iloc[0]
+    first_row = result.value.loc[result.value['Model'] != 'Perfect Model'].sort_values(by='Number of samples',
+                                                                                       ascending=False).iloc[0]
     # Assert
-    assert_that(len(result.value), equal_to(8))
+    assert_that(len(result.value), equal_to(12))
     assert_that(first_row['Value'], close_to(0.994, 0.05))
     assert_that(first_row['Number of samples'], equal_to(1135))
     assert_that(first_row['Class'], equal_to(1))
@@ -43,9 +46,10 @@ def test_mnist_most_frequent(mnist_dataset_train, mnist_dataset_test, mock_train
     # Act
     result = check.run(mnist_dataset_train, mnist_dataset_test, mock_trained_mnist,
                        device=device)
-    first_row = result.value.sort_values(by='Number of samples', ascending=False).iloc[0]
+    first_row = result.value.loc[result.value['Model'] != 'Perfect Model'].sort_values(by='Number of samples',
+                                                                                       ascending=False).iloc[0]
     # Assert
-    assert_that(len(result.value), equal_to(8))
+    assert_that(len(result.value), equal_to(12))
     assert_that(first_row['Value'], close_to(0.994, 0.05))
     assert_that(first_row['Number of samples'], equal_to(1135))
     assert_that(first_row['Class'], equal_to(1))
@@ -58,7 +62,7 @@ def test_mnist_uniform(mnist_dataset_train, mnist_dataset_test, mock_trained_mni
     result = check.run(mnist_dataset_train, mnist_dataset_test, mock_trained_mnist,
                        device=device)
     # Assert
-    assert_that(len(result.value), equal_to(8))
+    assert_that(len(result.value), equal_to(12))
 
 
 def test_mnist_stratified(mnist_dataset_train, mnist_dataset_test, mock_trained_mnist, device):
@@ -68,4 +72,51 @@ def test_mnist_stratified(mnist_dataset_train, mnist_dataset_test, mock_trained_
     result = check.run(mnist_dataset_train, mnist_dataset_test, mock_trained_mnist,
                        device=device)
     # Assert
-    assert_that(len(result.value), equal_to(8))
+    assert_that(len(result.value), equal_to(12))
+
+
+def test_condition_failed_for_multiclass(mnist_dataset_train, mnist_dataset_test, mock_trained_mnist, device):
+    train_ds, test_ds, clf = mnist_dataset_train, mnist_dataset_test, mock_trained_mnist
+    # Arrange
+    check = SimpleModelComparison().add_condition_gain_not_less_than(0.8)
+    # Act X
+    result = check.run(train_ds, test_ds, clf)
+    # Assert
+    assert_that(result.conditions_results, has_items(
+        equal_condition_result(
+            is_pass=False,
+            name='Model performance gain over simple model is not less than 80%',
+            details='Found metrics with gain below threshold: {\'Recall\': {1: \'-5000%\'}}')
+
+    ))
+
+
+def test_condition_pass_for_multiclass_avg(mnist_dataset_train, mnist_dataset_test, mock_trained_mnist, device):
+    train_ds, test_ds, clf = mnist_dataset_train, mnist_dataset_test, mock_trained_mnist
+    # Arrange
+    check = SimpleModelComparison().add_condition_gain_not_less_than(0.43, average=True)
+    # Act X
+    result = check.run(train_ds, test_ds, clf)
+    # Assert
+    assert_that(result.conditions_results, has_items(
+        equal_condition_result(
+            is_pass=True,
+            name='Model performance gain over simple model is not less than 43%')
+    ))
+
+
+def test_condition_pass_for_multiclass_avg_with_classes(mnist_dataset_train, mnist_dataset_test, mock_trained_mnist,
+                                                        device):
+    train_ds, test_ds, clf = mnist_dataset_train, mnist_dataset_test, mock_trained_mnist
+    # Arrange
+    check = SimpleModelComparison().add_condition_gain_not_less_than(1, average=True, classes=[0])
+    # Act X
+    result = check.run(train_ds, test_ds, clf)
+    # Assert
+    assert_that(result.conditions_results, has_items(
+        equal_condition_result(
+            is_pass=False,
+            name='Model performance gain over simple model is not less than 100% for classes [0]',
+            details='Found metrics with gain below threshold: {\'Recall\': \'99.39%\', \'Precision\': \'97.89%\'}'
+        )
+    ))