Repository: scikit-learn/scikit-learn
File: sklearn/metrics/tests/test_common.py (Python)

Code before:
    # the corresponding weights to zero
    sample_weight_subset = sample_weight[1::2]
    sample_weight_zeroed = np.copy(sample_weight)
    sample_weight_zeroed[::2] = 0
    y1_subset = y1[1::2]
    y2_subset = y2[1::2]
    weighted_score_subset = metric(
        y1_subset, y2_subset, sample_weight=sample_weight_subset
    )
    weighted_score_zeroed = metric(y1, y2, sample_weight=sample_weight_zeroed)
    assert_allclose(
        weighted_score_subset,
        weighted_score_zeroed,
        err_msg=(
            "Zeroing weights does not give the same result as "
            "removing the corresponding samples (%s != %s) for %s"
        )
        % (weighted_score_zeroed, weighted_score_subset, name),
    )

    # Check the score is invariant under scaling of weights by a constant factor
    # `confusion_matrix_at_thresholds` returns raw `tps`, `fps` etc values, which
    # are scaled by weights, so will vary e.g., scaling by 3 will result in 3 * `tps`
    if not (
        name.startswith("unnormalized") or name == "confusion_matrix_at_thresholds"
    ):
        # Numerical instability of floating points in `cumulative_sum` in
        # `median_absolute_error`, and in `diff` when in calculating collinear points
        # and points in between to drop `roc_curve` means they are not always
        # equivalent when scaling by a float.
        scaling_values = (
            [2] if name in {"median_absolute_error", "roc_curve"} else [2, 0.3]
        )
        for scaling in scaling_values:
            assert_allclose(
                weighted_score,
                metric(y1, y2, sample_weight=sample_weight * scaling),
                err_msg="%s sample_weight is not invariant under scaling" % name,
            )

    # Check that if number of samples in y_true and sample_weight are not
    # equal, meaningful error is raised.
    error_message = (
        r"Found input variables with inconsistent numbers of "
        r"samples: \[{}, {}, {}\]".format(
            _num_samples(y1), _num_samples(y2), _num_samples(sample_weight) * 2
        )
    )
    with pytest.raises(ValueError, match=error_message):
        metric(y1, y2, sample_weight=np.hstack([sample_weight, sample_weight]))


Review comment (suggestion):
I think we should define a test module constant `WEIGHT_SCALE_DEPENDENT_METRICS` with all the metric names that are expected to be dependent on the weight scale.

Code after:
        sample_weight=None,
    )
    assert_allclose(
        weighted_score,
        repeat_weighted_score,
        err_msg="Weighting %s is not equal to repeating samples" % name,
    )

    # check that ignoring a fraction of the samples is equivalent to setting
    # the corresponding weights to zero
    sample_weight_subset = sample_weight[1::2]
    sample_weight_zeroed = np.copy(sample_weight)
    sample_weight_zeroed[::2] = 0
    y1_subset = y1[1::2]
    y2_subset = y2[1::2]
    weighted_score_subset = metric(
        y1_subset, y2_subset, sample_weight=sample_weight_subset
    )
    weighted_score_zeroed = metric(y1, y2, sample_weight=sample_weight_zeroed)
    assert_allclose(
        weighted_score_subset,
        weighted_score_zeroed,
        err_msg=(
            "Zeroing weights does not give the same result as "
            "removing the corresponding samples (%s != %s) for %s"
        )
        % (weighted_score_zeroed, weighted_score_subset, name),
    )

    # Check the score is invariant under scaling of weights by a constant factor
    if name not in WEIGHT_SCALE_DEPENDENT_METRICS:
        # Numerical instability of floating points in `cumulative_sum` in
        # `median_absolute_error`, and in `diff` when in calculating collinear points
        # and points in between to drop `roc_curve` means they are not always
        # equivalent when scaling by a float.
        scaling_values = (
            [2] if name in {"median_absolute_error", "roc_curve"} else [2, 0.3]
        )
        for scaling in scaling_values:
            assert_allclose(
                weighted_score,
                metric(y1, y2, sample_weight=sample_weight * scaling),
                err_msg="%s sample_weight is not invariant under scaling" % name,
            )

    # Check that if number of samples in y_true and sample_weight are not
    # equal, meaningful error is raised.
    error_message = (
        r"Found input variables with inconsistent numbers of "
        r"samples: \[{}, {}, {}\]".format(
            _num_samples(y1), _num_samples(y2), _num_samples(sample_weight) * 2