Repository: deepchecks/deepchecks

Issue:
[BUG] using .head(2) to validate scorers fails when sample contains single label
This happens inside `validate_fitting` when should_return_array = False.

Running `self._run_score(model, df.head(2), dataset)` when head contains only single kind of label raises the following bug:

> TypeError: issubclass() arg 2 must be a class or tuple of classes



Discussion:

Resolving patch:
diff --git a/deepchecks/checks/performance/performance_report.py b/deepchecks/checks/performance/performance_report.py
--- a/deepchecks/checks/performance/performance_report.py
+++ b/deepchecks/checks/performance/performance_report.py
@@ -151,7 +151,7 @@ def add_condition_train_test_performance_degradation_ratio_not_greater_than(self
         """Add condition that will check that test performance is not degraded by more than given percentage in train.
 
         Args:
-            threshold: maximum degradation ratio allowed (value between 0 to 1)
+            threshold: maximum degradation ratio allowed (value between 0 and 1)
         """
         def condition(check_result: pd.DataFrame) -> ConditionResult:
             test_scores = check_result.loc[check_result['Dataset'] == 'Test']
diff --git a/deepchecks/utils/metrics.py b/deepchecks/utils/metrics.py
--- a/deepchecks/utils/metrics.py
+++ b/deepchecks/utils/metrics.py
@@ -137,7 +137,7 @@ def validate_fitting(self, model, dataset: 'base.Dataset', should_return_array:
         """Validate given scorer for the model and dataset."""
         df = self.filter_nulls(dataset)
         if should_return_array:
-            # In order for multiclass scorer to return array in right length need to pass him samples from all labels
+            # In order for scorer to return array in right length need to pass him samples from all labels
             single_label_data = df.groupby(dataset.label_name).head(1)
             result = self._run_score(model, single_label_data, dataset)
             if not isinstance(result, np.ndarray):
@@ -157,7 +157,9 @@ def validate_fitting(self, model, dataset: 'base.Dataset', should_return_array:
                                                   f'returned only {len(result)} elements in the score array value')
 
         else:
-            result = self._run_score(model, df.head(2), dataset)
+            # In order for scorer to run, it must have at least one sample of each label.
+            single_label_data = df.groupby(dataset.label_name).head(1)
+            result = self._run_score(model, single_label_data, dataset)
             if not isinstance(result, Number):
                 raise errors.DeepchecksValueError(f'Expected scorer {self.name} to return number '
                                                   f'but got: {type(result).__name__}')


Test patch:
diff --git a/tests/checks/performance/performance_report_test.py b/tests/checks/performance/performance_report_test.py
--- a/tests/checks/performance/performance_report_test.py
+++ b/tests/checks/performance/performance_report_test.py
@@ -14,7 +14,8 @@
 
 from hamcrest import assert_that, calling, raises, close_to, has_items
 import numpy as np
-from sklearn.ensemble import AdaBoostClassifier
+from sklearn.ensemble import AdaBoostClassifier, RandomForestClassifier
+from sklearn.model_selection import train_test_split
 
 from deepchecks import ConditionResult, Dataset
 from deepchecks.base.check import CheckResult
@@ -69,6 +70,26 @@ def test_classification(iris_split_dataset_and_model):
                 metric_col = class_col.loc[class_col['Metric'] == metric]
                 assert_that(metric_col['Value'] , close_to(1, 0.3))
 
+
+def test_classification_binary(iris_dataset_single_class_labeled):
+    # Arrange
+    train, test = train_test_split(iris_dataset_single_class_labeled.data, test_size=0.33, random_state=42)
+    train_ds = iris_dataset_single_class_labeled.copy(train)
+    test_ds = iris_dataset_single_class_labeled.copy(test)
+    clf = RandomForestClassifier(random_state=0)
+    clf.fit(train_ds.features_columns, train_ds.label_col)
+    check = PerformanceReport()
+
+    # Act X
+    result = check.run(train_ds, test_ds, clf).value
+    # Assert
+    for dataset in ['Test', 'Train']:
+        dataset_col = result.loc[result['Dataset'] == dataset]
+        for metric in dataset_col['Metric'].unique():
+            metric_col = dataset_col.loc[dataset_col['Metric'] == metric]
+            assert_that(metric_col['Value'], close_to(1, 0.3))
+
+
 def test_classification_string_labels(iris_labeled_dataset):
     # Arrange
     check = PerformanceReport()