Repository: scikit-learn/scikit-learn

Issue:
# Make `zero_division` parameter consistent in the different metric

This is an issue to report the step to actually take over the work of @marctorsoc in https://github.com/scikit-learn/scikit-learn/pull/23183 and split the PR into smaller one to facilitate the review process.

The intend is to make the `zero_division` parameter consistent across different metrics in scikit-learn. In this regards, we have the following TODO list:

- [x] Introduce the `zero_division` parameter to the `accuracy_score` function when `y_true` and `y_pred` are empty.
    - https://github.com/scikit-learn/scikit-learn/pull/29213
- [x] Introduce the `zero_division` parameter to the `class_likelihood_ratios` and remove `raise_warning`.
    - https://github.com/scikit-learn/scikit-learn/pull/31331
- [x] Introduce the `zero_division` parameter to the `cohen_kappa_score` function
  - https://github.com/scikit-learn/scikit-learn/pull/29210
- [x] Introduce the `zero_division` parameter to the `matthew_corr_coeff` function
  - #23183
  - #28509
- [ ] <del>Open a PR to make sure the empty input lead to `np.nan` in `classification_report` function.</del> `classification_report` should raise an error instead: see https://github.com/scikit-learn/scikit-learn/issues/29048#issuecomment-2857931743

All those items have been addressed in #23183 and can be extracted in individual PRs. The changelog presenting the changes should acknowledge @marctorsoc.

In addition, we should investigate #27047 and check if we should add the `zero_division` parameter to the `precision_recall_curve` and `roc_curve` as well. This might add two additional items to the list above.

Implementation patch:
diff --git a/sklearn/metrics/_classification.py b/sklearn/metrics/_classification.py
index 01d8b93a510d7..894f291eaa4e7 100644
--- a/sklearn/metrics/_classification.py
+++ b/sklearn/metrics/_classification.py
@@ -883,10 +883,22 @@ def multilabel_confusion_matrix(
         "labels": ["array-like", None],
         "weights": [StrOptions({"linear", "quadratic"}), None],
         "sample_weight": ["array-like", None],
+        "replace_undefined_by": [
+            Interval(Real, -1.0, 1.0, closed="both"),
+            np.nan,
+        ],
     },
     prefer_skip_nested_validation=True,
 )
-def cohen_kappa_score(y1, y2, *, labels=None, weights=None, sample_weight=None):
+def cohen_kappa_score(
+    y1,
+    y2,
+    *,
+    labels=None,
+    weights=None,
+    sample_weight=None,
+    replace_undefined_by=np.nan,
+):
     r"""Compute Cohen's kappa: a statistic that measures inter-annotator agreement.
 
     This function computes Cohen's kappa [1]_, a score that expresses the level
@@ -927,11 +939,25 @@ class labels [2]_.
     sample_weight : array-like of shape (n_samples,), default=None
         Sample weights.
 
+    replace_undefined_by : np.nan, float in [-1.0, 1.0], default=np.nan
+        Sets the return value when the metric is undefined. This can happen when no
+        label of interest (as defined in the `labels` param) is assigned by the second
+        annotator, or when both `y1` and `y2`only have one label in common that is also
+        in `labels`. In these cases, an
+        :class:`~sklearn.exceptions.UndefinedMetricWarning` is raised. Can take the
+        following values:
+
+        - `np.nan` to return `np.nan`
+        - a floating point value in the range of [-1.0, 1.0] to return a specific value
+
+        .. versionadded:: 1.9
+
     Returns
     -------
     kappa : float
-        The kappa statistic, which is a number between -1 and 1. The maximum
-        value means complete agreement; zero or lower means chance agreement.
+        The kappa statistic, which is a number between -1.0 and 1.0. The maximum value
+        means complete agreement; the minimum value means complete disagreement; 0.0
+        indicates no agreement beyond what would be expected by chance.
 
     References
     ----------
@@ -974,7 +1000,20 @@ class labels [2]_.
     confusion = xp.astype(confusion, max_float_dtype, copy=False)
     sum0 = xp.sum(confusion, axis=0)
     sum1 = xp.sum(confusion, axis=1)
-    expected = xp.linalg.outer(sum0, sum1) / xp.sum(sum0)
+
+    numerator = xp.linalg.outer(sum0, sum1)
+    denominator = xp.sum(sum0)
+    msg_zero_division = (
+        "`y2` contains no labels that are present in both `y1` and `labels`."
+        "`cohen_kappa_score` is undefined and set to the value defined by "
+        f"the `replace_undefined_by` param, which is set to {replace_undefined_by}."
+    )
+    # exact equality is safe here, since denominator is a sum of positive terms:
+    if denominator == 0:
+        warnings.warn(msg_zero_division, UndefinedMetricWarning, stacklevel=2)
+        return replace_undefined_by
+
+    expected = numerator / denominator
 
     if weights is None:
         w_mat = xp.ones([n_classes, n_classes], dtype=max_float_dtype, device=device_)
@@ -987,7 +1026,19 @@ class labels [2]_.
         else:
             w_mat = (w_mat - w_mat.T) ** 2
 
-    k = xp.sum(w_mat * confusion) / xp.sum(w_mat * expected)
+    numerator = xp.sum(w_mat * confusion)
+    denominator = xp.sum(w_mat * expected)
+    msg_zero_division = (
+        "`y1`, `y2` and `labels` have only one label in common. "
+        "`cohen_kappa_score` is undefined and set to the value defined by the "
+        f"the `replace_undefined_by` param, which is set to {replace_undefined_by}."
+    )
+    # exact equality is safe here, since denominator is a sum of positive terms:
+    if denominator == 0:
+        warnings.warn(msg_zero_division, UndefinedMetricWarning, stacklevel=2)
+        return replace_undefined_by
+
+    k = numerator / denominator
     return float(1 - k)
 
 


Test patch:
diff --git a/sklearn/metrics/tests/test_classification.py b/sklearn/metrics/tests/test_classification.py
index eb267ccf5e696..9a42b8a5acaf4 100644
--- a/sklearn/metrics/tests/test_classification.py
+++ b/sklearn/metrics/tests/test_classification.py
@@ -895,6 +895,68 @@ def test_cohen_kappa():
     )
 
 
+@ignore_warnings(category=UndefinedMetricWarning)
+@pytest.mark.parametrize(
+    "test_case",
+    [
+        # annotator y2 does not assign any label specified in `labels` (note: also
+        # applicable if `labels` is default and `y2` does not contain any label that is
+        # in `y1`):
+        ([1] * 5 + [2] * 5, [3] * 10, [1, 2], None),
+        # both inputs (`y1` and `y2`) only have one label:
+        ([3] * 10, [3] * 10, None, None),
+        # both inputs only have one label in common that is also in `labels`:
+        ([1] * 5 + [2] * 5, [1] * 5 + [3] * 5, [1, 2], None),
+        # like the last test case, but with `weights="linear"` (note that
+        # weights="linear" and weights="quadratic" are different branches, though the
+        # latter is so similar to the former that the test case is skipped here):
+        ([1] * 5 + [2] * 5, [1] * 5 + [3] * 5, [1, 2], "linear"),
+    ],
+)
+@pytest.mark.parametrize("replace_undefined_by", [0.0, np.nan])
+def test_cohen_kappa_undefined(test_case, replace_undefined_by):
+    """Test that cohen_kappa_score handles divisions by 0 correctly by returning the
+    `replace_undefined_by` param. (The first test case covers the first possible
+    location in the function for an occurrence of a division by zero, the last three
+    test cases cover a zero division in the the second possible location in the
+    function."""
+
+    y1, y2, labels, weights = test_case
+    y1, y2 = np.array(y1), np.array(y2)
+
+    score = cohen_kappa_score(
+        y1,
+        y2,
+        labels=labels,
+        weights=weights,
+        replace_undefined_by=replace_undefined_by,
+    )
+    assert_allclose(score, replace_undefined_by, equal_nan=True)
+
+
+def test_cohen_kappa_zero_division_warning():
+    """Test that cohen_kappa_score raises UndefinedMetricWarning when a division by 0
+    occurs."""
+
+    labels = [1, 2]
+    y1 = np.array([1] * 5 + [2] * 5)
+    y2 = np.array([3] * 10)
+    with pytest.warns(
+        UndefinedMetricWarning,
+        match="`y2` contains no labels that are present in both `y1` and `labels`.",
+    ):
+        cohen_kappa_score(y1, y2, labels=labels)
+
+    labels = [1, 2]
+    y1 = np.array([1] * 5 + [2] * 5)
+    y2 = np.array([1] * 5 + [3] * 5)
+    with pytest.warns(
+        UndefinedMetricWarning,
+        match="`y1`, `y2` and `labels` have only one label in common.",
+    ):
+        cohen_kappa_score(y1, y2, labels=labels)
+
+
 def test_cohen_kappa_score_error_wrong_label():
     """Test that correct error is raised when users pass labels that are not in y1."""
     labels = [1, 2]