Repository: deepchecks/deepchecks

Issue:
[BUG] some checks/condition have unwanted behavior if test contains new classes
**Describe the bug**
some checks/condition have unwanted behavior if test contains new classes

**Expected behavior**
need to discuss 

**Screenshots**
![Screen Shot 2022-03-28 at 16 24 20](https://user-images.githubusercontent.com/26321553/160407823-a044a5de-7262-4e33-8a98-35e22ae836eb.png)



Discussion:

Resolving patch:
diff --git a/deepchecks/tabular/checks/performance/simple_model_comparison.py b/deepchecks/tabular/checks/performance/simple_model_comparison.py
--- a/deepchecks/tabular/checks/performance/simple_model_comparison.py
+++ b/deepchecks/tabular/checks/performance/simple_model_comparison.py
@@ -140,7 +140,7 @@ def run_logic(self, context: Context) -> CheckResult:
         # Multiclass have different return type from the scorer, list of score per class instead of single score
         if task_type in [ModelType.MULTICLASS, ModelType.BINARY]:
             n_samples = test_label.groupby(test_label).count()
-            classes = test_dataset.classes
+            classes = [clazz for clazz in test_dataset.classes if clazz in train_dataset.classes]
 
             results_array = []
             # Dict in format { Scorer : Dict { Class : Dict { Origin/Simple : score } } }
diff --git a/deepchecks/vision/checks/performance/simple_model_comparison.py b/deepchecks/vision/checks/performance/simple_model_comparison.py
--- a/deepchecks/vision/checks/performance/simple_model_comparison.py
+++ b/deepchecks/vision/checks/performance/simple_model_comparison.py
@@ -131,8 +131,8 @@ def update(self, context: Context, batch: Batch, dataset_kind: DatasetKind):
                 metric.update((prediction, label))
 
             # calculating perfect scores
-            n_of_classes = batch.predictions.to('cpu').shape[1]
-            perfect_predictions = np.eye(n_of_classes)[label.to('cpu').numpy()]
+            n_of_classes = batch.predictions.cpu().detach().shape[1]
+            perfect_predictions = np.eye(n_of_classes)[label.cpu().detach().numpy()]
             for _, metric in self._perfect_metrics.items():
                 metric.update((torch.Tensor(perfect_predictions).to(context.device), label))
 
@@ -171,6 +171,7 @@ def compute(self, context: Context) -> CheckResult:
                                                          column_filter_value='Given Model')
             results_df = results_df.loc[results_df['Class'].isin(classes_to_show)]
 
+        results_df = results_df.dropna()
         results_df = results_df.sort_values(by=['Model', 'Value'], ascending=False).reset_index(drop=True)
 
         fig = px.histogram(


Test patch:
diff --git a/tests/checks/performance/simple_model_comparison_test.py b/tests/checks/performance/simple_model_comparison_test.py
--- a/tests/checks/performance/simple_model_comparison_test.py
+++ b/tests/checks/performance/simple_model_comparison_test.py
@@ -169,6 +169,21 @@ def test_condition_pass_for_multiclass_avg_with_classes(iris_split_dataset_and_m
     ))
 
 
+def test_condition_pass_for_new_test_classes(city_arrogance_split_dataset_and_model):
+    train_ds, test_ds, clf = city_arrogance_split_dataset_and_model
+    # Arrange
+    check = SimpleModelComparison(simple_model_type='constant').add_condition_gain_not_less_than(1)
+    # Act X
+    result = check.run(train_ds, test_ds, clf)
+    # Assert
+    assert_that(result.conditions_results, has_items(
+        equal_condition_result(
+            is_pass=True,
+            name='Model performance gain over simple model is not less than 100%',
+        )
+    ))
+
+
 def test_condition_ratio_not_less_than_passed(diabetes_split_dataset_and_model):
     # Arrange
     train_ds, test_ds, clf = diabetes_split_dataset_and_model