Repository: deepchecks/deepchecks

Issue:
Metrics: change binary model to support metric per class
1. When using `multiclass_avg=False` in the metrics functions for binary model, need to return the multiclass scorers instead of the binary ones. 
2. In checks using `multiclass_avg=False` make the necessary updates


Discussion:

Resolving patch:
diff --git a/deepchecks/checks/performance/performance_report.py b/deepchecks/checks/performance/performance_report.py
--- a/deepchecks/checks/performance/performance_report.py
+++ b/deepchecks/checks/performance/performance_report.py
@@ -103,7 +103,7 @@ def _performance_report(self, train_dataset: Dataset, test_dataset: Dataset, mod
         scorers = get_scorers_list(model, test_dataset, self.alternative_scorers, multiclass_avg=False)
         datasets = {'Train': train_dataset, 'Test': test_dataset}
 
-        if task_type == ModelType.MULTICLASS:
+        if task_type in [ModelType.MULTICLASS, ModelType.BINARY]:
             plot_x_axis = ['Class', 'Dataset']
             results = []
 
@@ -139,7 +139,7 @@ def _performance_report(self, train_dataset: Dataset, test_dataset: Dataset, mod
             hover_data=['Number of samples']
         )
 
-        if task_type == ModelType.MULTICLASS:
+        if task_type in [ModelType.MULTICLASS, ModelType.BINARY]:
             fig.update_xaxes(tickprefix='Class ', tickangle=60)
 
         fig = (
@@ -308,7 +308,7 @@ def run_logic(self, context: ModelComparisonContext):
         first_test_ds = context.test_datasets[0]
         scorers = get_scorers_list(first_model, first_test_ds, self.alternative_scorers, multiclass_avg=False)
 
-        if context.task_type == ModelType.MULTICLASS:
+        if context.task_type in [ModelType.MULTICLASS, ModelType.BINARY]:
             plot_x_axis = ['Class', 'Model']
             results = []
 
@@ -344,7 +344,7 @@ def run_logic(self, context: ModelComparisonContext):
             hover_data=['Number of samples'],
         )
 
-        if context.task_type == ModelType.MULTICLASS:
+        if context.task_type in [ModelType.MULTICLASS, ModelType.BINARY]:
             fig.update_xaxes(title=None, tickprefix='Class ', tickangle=60)
         else:
             fig.update_xaxes(title=None)
diff --git a/deepchecks/checks/performance/simple_model_comparison.py b/deepchecks/checks/performance/simple_model_comparison.py
--- a/deepchecks/checks/performance/simple_model_comparison.py
+++ b/deepchecks/checks/performance/simple_model_comparison.py
@@ -126,7 +126,7 @@ def run(self, train_dataset, test_dataset, model) -> CheckResult:
         ]
 
         # Multiclass have different return type from the scorer, list of score per class instead of single score
-        if task_type == ModelType.MULTICLASS:
+        if task_type in [ModelType.MULTICLASS, ModelType.BINARY]:
             results = []
             for model_name, model_type, model_instance in models:
                 for scorer in scorers:
diff --git a/deepchecks/utils/metrics.py b/deepchecks/utils/metrics.py
--- a/deepchecks/utils/metrics.py
+++ b/deepchecks/utils/metrics.py
@@ -32,6 +32,9 @@
     'initialize_single_scorer',
     'DEFAULT_SCORERS_DICT',
     'DEFAULT_SINGLE_SCORER',
+    'DEFAULT_REGRESSION_SCORERS',
+    'DEFAULT_BINARY_SCORERS',
+    'DEFAULT_MULTICLASS_SCORERS',
     'MULTICLASS_SCORERS_NON_AVERAGE',
     'get_scores_ratio',
     'initialize_multi_scorers',
@@ -252,7 +255,7 @@ def get_scorers_list(
     """
     # Check for model type
     model_type = task_type_check(model, dataset)
-    multiclass_array = model_type == ModelType.MULTICLASS and multiclass_avg is False
+    multiclass_array = model_type in [ModelType.MULTICLASS, ModelType.BINARY] and multiclass_avg is False
 
     if alternative_scorers:
         scorers = alternative_scorers
@@ -274,7 +277,7 @@ def get_scorer_single(model, dataset: 'base.Dataset', alternative_scorer: t.Opti
                       multiclass_avg: bool = True) -> 'DeepcheckScorer':
     """Return single score to use in check, and validate scorer fit the model and dataset."""
     model_type = task_type_check(model, dataset)
-    multiclass_array = model_type == ModelType.MULTICLASS and multiclass_avg is False
+    multiclass_array = model_type in [ModelType.MULTICLASS, ModelType.BINARY] and multiclass_avg is False
 
     if alternative_scorer is None:
         if multiclass_array:


Test patch:
diff --git a/tests/checks/performance/performance_report_test.py b/tests/checks/performance/performance_report_test.py
--- a/tests/checks/performance/performance_report_test.py
+++ b/tests/checks/performance/performance_report_test.py
@@ -11,18 +11,18 @@
 """Contains unit tests for the performance report check."""
 import re
 from typing import List
-
-from hamcrest import assert_that, calling, raises, close_to, has_items
 import numpy as np
+
+from hamcrest import assert_that, calling, raises, close_to, has_items, instance_of
 from sklearn.ensemble import AdaBoostClassifier, RandomForestClassifier
 from sklearn.model_selection import train_test_split
 
 from deepchecks import ConditionResult, Dataset
-from deepchecks.base.check import CheckResult
 from deepchecks.checks.performance import PerformanceReport
 from deepchecks.errors import DeepchecksValueError
 
 from tests.checks.utils import equal_condition_result
+from deepchecks.utils.metrics import MULTICLASS_SCORERS_NON_AVERAGE, DEFAULT_REGRESSION_SCORERS
 
 
 def test_dataset_wrong_input():
@@ -55,6 +55,17 @@ def test_dataset_no_shared_label(iris_labeled_dataset):
                 raises(DeepchecksValueError,
                        'Check requires datasets to share the same label'))
 
+
+def assert_classification_result(result, dataset: Dataset):
+    for dataset_name in ['Test', 'Train']:
+        dataset_df = result.loc[result['Dataset'] == dataset_name]
+        for class_name in dataset.classes:
+            class_df = dataset_df.loc[dataset_df['Class'] == class_name]
+            for metric in MULTICLASS_SCORERS_NON_AVERAGE.keys():
+                metric_row = class_df.loc[class_df['Metric'] == metric]
+                assert_that(metric_row['Value'].iloc[0], close_to(1, 0.3))
+
+
 def test_classification(iris_split_dataset_and_model):
     # Arrange
     train, test, model = iris_split_dataset_and_model
@@ -62,13 +73,7 @@ def test_classification(iris_split_dataset_and_model):
     # Act X
     result = check.run(train, test, model).value
     # Assert
-    for dataset in ['Test', 'Train']:
-        dataset_col = result.loc[result['Dataset'] == dataset]
-        for class_name in range(3):
-            class_col = dataset_col.loc[dataset_col['Class'] == class_name]
-            for metric in ['F1 (Default)', 'Precision (Default)', 'Recall (Default)']:
-                metric_col = class_col.loc[class_col['Metric'] == metric]
-                assert_that(metric_col['Value'] , close_to(1, 0.3))
+    assert_classification_result(result, test)
 
 
 def test_classification_binary(iris_dataset_single_class_labeled):
@@ -83,11 +88,7 @@ def test_classification_binary(iris_dataset_single_class_labeled):
     # Act X
     result = check.run(train_ds, test_ds, clf).value
     # Assert
-    for dataset in ['Test', 'Train']:
-        dataset_col = result.loc[result['Dataset'] == dataset]
-        for metric in dataset_col['Metric'].unique():
-            metric_col = dataset_col.loc[dataset_col['Metric'] == metric]
-            assert_that(metric_col['Value'], close_to(1, 0.3))
+    assert_classification_result(result, test_ds)
 
 
 def test_classification_string_labels(iris_labeled_dataset):
@@ -102,13 +103,7 @@ def test_classification_string_labels(iris_labeled_dataset):
     # Act X
     result = check.run(iris_labeled_dataset, iris_labeled_dataset, iris_adaboost).value
     # Assert
-    for dataset in ['Test', 'Train']:
-        dataset_col = result.loc[result['Dataset'] == dataset]
-        for class_name in iris_labeled_dataset.classes:
-            class_col = dataset_col.loc[dataset_col['Class'] == class_name]
-            for metric in ['F1 (Default)', 'Precision (Default)', 'Recall (Default)']:
-                metric_col = class_col.loc[class_col['Metric'] == metric]
-                assert_that(metric_col['Value'] , close_to(1, 0.3))
+    assert_classification_result(result, iris_labeled_dataset)
 
 
 def test_classification_nan_labels(iris_labeled_dataset, iris_adaboost):
@@ -121,13 +116,7 @@ def test_classification_nan_labels(iris_labeled_dataset, iris_adaboost):
     # Act X
     result = check.run(iris_labeled_dataset, iris_labeled_dataset, iris_adaboost).value
     # Assert
-    for dataset in ['Test', 'Train']:
-        dataset_col = result.loc[result['Dataset'] == dataset]
-        for class_name in iris_labeled_dataset.classes:
-            class_col = dataset_col.loc[dataset_col['Class'] == class_name]
-            for metric in ['F1 (Default)', 'Precision (Default)', 'Recall (Default)']:
-                metric_col = class_col.loc[class_col['Metric'] == metric]
-                assert_that(metric_col['Value'] , close_to(1, 0.3))
+    assert_classification_result(result, iris_labeled_dataset)
 
 
 def test_regression(diabetes_split_dataset_and_model):
@@ -139,9 +128,9 @@ def test_regression(diabetes_split_dataset_and_model):
     # Assert
     for dataset in ['Test', 'Train']:
         dataset_col = result.loc[result['Dataset'] == dataset]
-        for metric in ['Neg RMSE (Default)', 'Neg MAE (Default)']:
+        for metric in DEFAULT_REGRESSION_SCORERS.keys():
             metric_col = dataset_col.loc[dataset_col['Metric'] == metric]
-            assert_that(metric_col['Value'] , close_to(-50, 30))
+            assert_that(metric_col['Value'].iloc[0], instance_of(float))
 
 
 def test_condition_min_score_not_passed(iris_split_dataset_and_model):