Repository: scikit-learn/scikit-learn
File: sklearn/metrics/_classification.py (Python)

Code before:
    elif sample_weight is not None:
        tn = xp.sum(sample_weight) - tp - fp - fn
    elif samplewise:
        tn = y_true.shape[1] - tp - fp - fn
    else:
        tn = y_true.shape[0] - tp - fp - fn

    return xp.reshape(xp.stack([tn, fp, fn, tp]).T, (-1, 2, 2))


@validate_params(
    {
        "y1": ["array-like"],
        "y2": ["array-like"],
        "labels": ["array-like", None],
        "weights": [StrOptions({"linear", "quadratic"}), None],
        "sample_weight": ["array-like", None],
        "replace_undefined_by": [
            Interval(Real, -1.0, 1.0, closed="both"),
            np.nan,
            Hidden(StrOptions({"deprecated"})),
        ],
    },
    prefer_skip_nested_validation=True,
)
# TODO(1.9): Change default value for `replace_undefined_by` param to 0.0 and remove
# FutureWarnings; also the defaults in the warning messages need to be updated.
def cohen_kappa_score(
    y1,
    y2,
    *,
    labels=None,
    weights=None,
    sample_weight=None,
    replace_undefined_by="deprecated",
):
    r"""Compute Cohen's kappa: a statistic that measures inter-annotator agreement.

    This function computes Cohen's kappa [1]_, a score that expresses the level
    of agreement between two annotators on a classification problem. It is
    defined as

    .. math::
        \kappa = (p_o - p_e) / (1 - p_e)

    where :math:`p_o` is the empirical probability of agreement on the label
    assigned to any sample (the observed agreement ratio), and :math:`p_e` is
    the expected agreement when both annotators assign labels randomly.
    :math:`p_e` is estimated using a per-annotator empirical prior over the
    class labels [2]_.


Review comment (suggestion):
```suggestion
# TODO(1.10): Change default value for `replace_undefined_by` param to 0.0 and remove
```

Code after:
                )

        if n_labels is not None:
            y_true = y_true[:, labels[:n_labels]]
            y_pred = y_pred[:, labels[:n_labels]]

        if issparse(y_true) or issparse(y_pred):
            true_and_pred = y_true.multiply(y_pred)
        else:
            true_and_pred = xp.multiply(y_true, y_pred)

        # calculate weighted counts
        tp_sum = _count_nonzero(
            true_and_pred,
            axis=sum_axis,
            sample_weight=sample_weight,
            xp=xp,
            device=device_,
        )
        pred_sum = _count_nonzero(
            y_pred,
            axis=sum_axis,
            sample_weight=sample_weight,
            xp=xp,
            device=device_,
        )
        true_sum = _count_nonzero(
            y_true,
            axis=sum_axis,
            sample_weight=sample_weight,
            xp=xp,
            device=device_,
        )

    fp = pred_sum - tp_sum
    fn = true_sum - tp_sum
    tp = tp_sum

    if sample_weight is not None and samplewise:
        tp = xp.asarray(tp)
        fp = xp.asarray(fp)
        fn = xp.asarray(fn)
        tn = sample_weight * y_true.shape[1] - tp - fp - fn
    elif sample_weight is not None:
        tn = xp.sum(sample_weight) - tp - fp - fn
    elif samplewise:
        tn = y_true.shape[1] - tp - fp - fn
    else:
        tn = y_true.shape[0] - tp - fp - fn

    return xp.reshape(xp.stack([tn, fp, fn, tp]).T, (-1, 2, 2))