Repository: scikit-learn/scikit-learn
File: sklearn/metrics/tests/test_common.py (Python)

Code before:
    weighted_score_zeroed = metric(y1, y2, sample_weight=sample_weight_zeroed)
    assert_allclose(
        weighted_score_subset,
        weighted_score_zeroed,
        err_msg=(
            "Zeroing weights does not give the same result as "
            "removing the corresponding samples (%s != %s) for %s"
        )
        % (weighted_score_zeroed, weighted_score_subset, name),
    )

    # Check the score is invariant under scaling of weights by a constant factor
    # `confusion_matrix_at_thresholds` returns raw `tps`, `fps` etc values, which
    # are scaled by weights, so will vary e.g., scaling by 3 will result in 3 * `tps`
    if not (
        name.startswith("unnormalized") or name == "confusion_matrix_at_thresholds"
    ):
        # Numerical instability of floating points in `cumulative_sum` in
        # `median_absolute_error`, and in `diff` when in calculating collinear points
        # and points in between to drop `roc_curve` means they are not always
        # equivalent when scaling by a float.
        scaling_values = (
            [2] if name in {"median_absolute_error", "roc_curve"} else [2, 0.3]
        )
        for scaling in scaling_values:
            assert_allclose(
                weighted_score,
                metric(y1, y2, sample_weight=sample_weight * scaling),
                err_msg="%s sample_weight is not invariant under scaling" % name,
            )

    # Check that if number of samples in y_true and sample_weight are not
    # equal, meaningful error is raised.
    error_message = (
        r"Found input variables with inconsistent numbers of "
        r"samples: \[{}, {}, {}\]".format(
            _num_samples(y1), _num_samples(y2), _num_samples(sample_weight) * 2
        )
    )
    with pytest.raises(ValueError, match=error_message):
        metric(y1, y2, sample_weight=np.hstack([sample_weight, sample_weight]))


@pytest.mark.parametrize(
    "name",
    sorted(
        set(ALL_METRICS).intersection(set(REGRESSION_METRICS))
        - METRICS_WITHOUT_SAMPLE_WEIGHT
    ),
)
def test_regression_sample_weight_invariance(name):

Review comment (suggestion):
In `roc_curve` we do:

https://github.com/scikit-learn/scikit-learn/blob/7f0900c265936eac9a89bba37eb19ee66208d46a/sklearn/metrics/_ranking.py#L1271-L1280

when scaling by a float, `diff` can result in extra indices. Note that in `confusion_matrix_at_thresholds` we already use max floating point precision when calculating `tps` and `fps` with `cumulative_sum`.

Also note `drop_intermediate=True` is the default.

<details open>
<summary>Details</summary>

In the test `test_binary_sample_weight_invariance`, with no scaling the indices are:

```
optimal_idxs=array([ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 15, 16, 17,
18, 20, 21, 22, 23, 24, 25, 26, 29, 30, 32, 33, 34, 35, 36, 37, 38,
39, 41, 42, 43, 44, 45, 46, 48, 49])
```

with scaling 0.3, the indices are:

```
optimal_idxs=array([ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,
17, 18, 20, 21, 22, 23, 24, 25, 26, 29, 30, 32, 33, 34, 35, 36, 37,
38, 39, 40, 41, 42, 43, 44, 45, 46, 48, 49])
```

there are extra indices: 14 and 40 here

</details>

Code after:
    # the corresponding weights to zero
    sample_weight_subset = sample_weight[1::2]
    sample_weight_zeroed = np.copy(sample_weight)
    sample_weight_zeroed[::2] = 0
    y1_subset = y1[1::2]
    y2_subset = y2[1::2]
    weighted_score_subset = metric(
        y1_subset, y2_subset, sample_weight=sample_weight_subset
    )
    weighted_score_zeroed = metric(y1, y2, sample_weight=sample_weight_zeroed)
    assert_allclose(
        weighted_score_subset,
        weighted_score_zeroed,
        err_msg=(
            "Zeroing weights does not give the same result as "
            "removing the corresponding samples (%s != %s) for %s"
        )
        % (weighted_score_zeroed, weighted_score_subset, name),
    )

    # Check the score is invariant under scaling of weights by a constant factor
    if name not in WEIGHT_SCALE_DEPENDENT_METRICS:
        # Numerical instability of floating points in `cumulative_sum` in
        # `median_absolute_error`, and in `diff` when in calculating collinear points
        # and points in between to drop `roc_curve` means they are not always
        # equivalent when scaling by a float.
        scaling_values = (
            [2] if name in {"median_absolute_error", "roc_curve"} else [2, 0.3]
        )
        for scaling in scaling_values:
            assert_allclose(
                weighted_score,
                metric(y1, y2, sample_weight=sample_weight * scaling),
                err_msg="%s sample_weight is not invariant under scaling" % name,
            )

    # Check that if number of samples in y_true and sample_weight are not
    # equal, meaningful error is raised.
    error_message = (
        r"Found input variables with inconsistent numbers of "
        r"samples: \[{}, {}, {}\]".format(
            _num_samples(y1), _num_samples(y2), _num_samples(sample_weight) * 2
        )
    )
    with pytest.raises(ValueError, match=error_message):
        metric(y1, y2, sample_weight=np.hstack([sample_weight, sample_weight]))


@pytest.mark.parametrize(
    "name",
    sorted(