Repository: deepchecks/deepchecks

Issue:
[FEAT] RocReport: Mark for each class optimal cutoff
Add a point mark on each class in the plot


Discussion:

Resolving patch:
diff --git a/deepchecks/checks/performance/roc_report.py b/deepchecks/checks/performance/roc_report.py
--- a/deepchecks/checks/performance/roc_report.py
+++ b/deepchecks/checks/performance/roc_report.py
@@ -26,7 +26,9 @@
 
 
 class RocReport(SingleDatasetBaseCheck):
-    """Calculate the AUC (Area Under Curve) for each class.
+    """Calculate the ROC curve for each class.
+
+    For each class plots the ROC curve, calculate AUC score and displays the optimal threshold cutoff point.
 
     Args:
         excluded_classes (List): List of classes to exclude from the condition.
@@ -63,32 +65,36 @@ def _roc_report(self, dataset: Dataset, model):
 
         fpr = {}
         tpr = {}
+        thresholds = {}
         roc_auc = {}
         for i, class_name in enumerate(dataset_classes):
             if class_name in self.excluded_classes:
                 continue
-            fpr[i], tpr[i], _ = sklearn.metrics.roc_curve(multi_y[:, i], y_pred_prob[:, i])
-            roc_auc[i] = sklearn.metrics.auc(fpr[i], tpr[i])
+            fpr[class_name], tpr[class_name], thresholds[class_name] = \
+                sklearn.metrics.roc_curve(multi_y[:, i], y_pred_prob[:, i])
+            roc_auc[class_name] = sklearn.metrics.auc(fpr[class_name], tpr[class_name])
 
         fig = go.Figure()
-        for i, class_name in enumerate(dataset_classes):
+        for class_name in dataset_classes:
             if class_name in self.excluded_classes:
                 continue
             if len(dataset_classes) == 2:
                 fig.add_trace(go.Scatter(
-                    x=fpr[i],
-                    y=tpr[i],
+                    x=fpr[class_name],
+                    y=tpr[class_name],
                     line_width=2,
-                    name=f'auc = {roc_auc[i]:0.2f}',
+                    name=f'auc = {roc_auc[class_name]:0.2f}',
                 ))
+                fig.add_trace(get_cutoff_figure(tpr[class_name], fpr[class_name], thresholds[class_name]))
                 break
             else:
                 fig.add_trace(go.Scatter(
-                    x=fpr[i],
-                    y=tpr[i],
+                    x=fpr[class_name],
+                    y=tpr[class_name],
                     line_width=2,
-                    name=f'Class {class_name} (auc = {roc_auc[i]:0.2f})',
+                    name=f'Class {class_name} (auc = {roc_auc[class_name]:0.2f})'
                 ))
+                fig.add_trace(get_cutoff_figure(tpr[class_name], fpr[class_name], thresholds[class_name], class_name))
         fig.add_trace(go.Scatter(
                     x=[0, 1],
                     y=[0, 1],
@@ -105,7 +111,12 @@ def _roc_report(self, dataset: Dataset, model):
             fig.update_layout(title_text='Receiver operating characteristic for multi-class data',
                               width=900, height=500)
 
-        return CheckResult(roc_auc, header='ROC Report', display=fig)
+        footnote = """<span style="font-size:0.8em"><i>
+        The marked points are the optimal threshold cut-off points. They are determined using Youden's index defined
+        as sensitivity + specificity - 1
+        </i></span>"""
+
+        return CheckResult(roc_auc, header='ROC Report', display=[fig, footnote])
 
     def add_condition_auc_not_less_than(self, min_auc: float = 0.7):
         """Add condition - require min allowed AUC score per class.
@@ -132,3 +143,39 @@ def condition(result: Dict) -> ConditionResult:
             suffix = ''
         return self.add_condition(f'Not less than {min_auc} AUC score for all the classes{suffix}',
                                   condition)
+
+
+def get_cutoff_figure(tpr, fpr, thresholds, class_name=None):
+    index = sensitivity_specificity_cutoff(tpr, fpr)
+    hovertemplate = 'TPR: %{y:.2%}<br>FPR: %{x:.2%}' + f'<br>Youden\'s Index: {thresholds[index]:.3}'
+    if class_name:
+        hovertemplate += f'<br>Class: {class_name}'
+    return go.Scatter(x=[fpr[index]], y=[tpr[index]], mode='markers', marker_size=15,
+                      hovertemplate=hovertemplate, showlegend=False)
+
+
+def sensitivity_specificity_cutoff(tpr, fpr):
+    """Find index of optimal cutoff point on curve.
+
+    Cut-off is determined using Youden's index defined as sensitivity + specificity - 1.
+
+    Parameters
+    ----------
+    tpr : array, shape = [n_roc_points]
+        True positive rate per threshold
+    fpr : array, shape = [n_roc_points]
+        False positive rate per threshold
+
+    References
+    ----------
+    Ewald, B. (2006). Post hoc choice of cut points introduced bias to diagnostic research.
+    Journal of clinical epidemiology, 59(8), 798-801.
+
+    Steyerberg, E.W., Van Calster, B., & Pencina, M.J. (2011). Performance measures for
+    prediction models and markers: evaluation of predictions and classifications.
+    Revista Espanola de Cardiologia (English Edition), 64(9), 788-794.
+
+    Jiménez-Valverde, A., & Lobo, J.M. (2007). Threshold criteria for conversion of probability
+    of species presence to either–or presence–absence. Acta oecologica, 31(3), 361-369.
+    """
+    return np.argmax(tpr - fpr)


Test patch:
diff --git a/tests/checks/performance/roc_report_test.py b/tests/checks/performance/roc_report_test.py
--- a/tests/checks/performance/roc_report_test.py
+++ b/tests/checks/performance/roc_report_test.py
@@ -13,7 +13,7 @@
 import pandas as pd
 from sklearn.linear_model import LogisticRegression
 from sklearn.model_selection import train_test_split
-from hamcrest import assert_that, calling, raises, has_items
+from hamcrest import assert_that, calling, raises, has_items, has_entries, has_length, close_to
 
 from deepchecks.base import Dataset
 from deepchecks.checks.performance import RocReport
@@ -43,6 +43,16 @@ def test_regresion_model(diabetes_split_dataset_and_model):
                                            r' \[\'multiclass\', \'binary\'\], but received model of type: regression'))
 
 
+def test_binary_classification(iris_binary_string_split_dataset_and_model):
+    # Arrange
+    train, _, clf = iris_binary_string_split_dataset_and_model
+    # Act
+    result = RocReport().run(train, clf).value
+    # Assert
+    assert_that(result, has_length(2))
+    assert_that(result, has_entries(a=close_to(0.9, 0.1), b=close_to(0.9, 0.1)))
+
+
 def test_model_info_object(iris_labeled_dataset, iris_adaboost):
     # Arrange
     check = RocReport()