Repository: scikit-learn/scikit-learn
File: sklearn/utils/tests/test_estimator_checks.py (Python)

Code before:
    # some others might also be skipped, if their dependencies are not installed.
    assert set(expected_to_fail.keys()) <= set(skipped_checks)


@_mark_thread_unsafe_if_pytest_imported
def test_xfail_count_with_no_fast_fail():
    """Test that the right number of xfail warnings are raised when on_fail is "warn".

    It also checks the number of raised EstimatorCheckFailedWarning, and checks the
    output of check_estimator.
    """
    est = NuSVC()
    expected_failed_checks = _get_expected_failed_checks(est)
    # This is to make sure we test a class that has some expected failures
    assert len(expected_failed_checks) > 0
    with warnings.catch_warnings(record=True) as records:
        logs = check_estimator(
            est,
            expected_failed_checks=expected_failed_checks,
            on_fail="warn",
        )
    xfail_warns = [
        w
        for w in records
        if w.category != SkipTestWarning and w.category != FutureWarning
    ]

    assert all([rec.category == EstimatorCheckFailedWarning for rec in xfail_warns])
    assert len(xfail_warns) == len(expected_failed_checks)

    xfailed = [log for log in logs if log["status"] == "xfail"]
    assert len(xfailed) == len(expected_failed_checks)


def test_check_estimator_callback():
    """Test that the callback is called with the right arguments."""
    call_count = {"xfail": 0, "skipped": 0, "passed": 0, "failed": 0}

    def callback(
        *,
        estimator,
        check_name,
        exception,
        status,
        expected_to_fail,
        expected_to_fail_reason,
    ):
        assert status in ("xfail", "skipped", "passed", "failed")
        nonlocal call_count
        call_count[status] += 1


Review comment (suggestion):
Rather than doing this, we should probably replace `est` by an instance of a class that does not raise any `FutureWarning` while running the common tests. `LogisticRegression` or `Ridge` might work for that purpose.

Code after:
    # some others might also be skipped, if their dependencies are not installed.
    assert set(expected_to_fail.keys()) <= set(skipped_checks)


@_mark_thread_unsafe_if_pytest_imported
def test_xfail_count_with_no_fast_fail():
    """Test that the right number of xfail warnings are raised when on_fail is "warn".

    It also checks the number of raised EstimatorCheckFailedWarning, and checks the
    output of check_estimator.
    """
    est = NuSVC()
    expected_failed_checks = _get_expected_failed_checks(est)
    # This is to make sure we test a class that has some expected failures
    assert len(expected_failed_checks) > 0
    with warnings.catch_warnings(record=True) as records:
        logs = check_estimator(
            est,
            expected_failed_checks=expected_failed_checks,
            on_fail="warn",
        )
    xfail_warns = [w for w in records if w.category != SkipTestWarning]
    assert all([rec.category == EstimatorCheckFailedWarning for rec in xfail_warns])
    assert len(xfail_warns) == len(expected_failed_checks)

    xfailed = [log for log in logs if log["status"] == "xfail"]
    assert len(xfailed) == len(expected_failed_checks)


def test_check_estimator_callback():
    """Test that the callback is called with the right arguments."""
    call_count = {"xfail": 0, "skipped": 0, "passed": 0, "failed": 0}

    def callback(
        *,
        estimator,
        check_name,
        exception,
        status,
        expected_to_fail,
        expected_to_fail_reason,
    ):
        assert status in ("xfail", "skipped", "passed", "failed")
        nonlocal call_count
        call_count[status] += 1

    est = NuSVC()
    expected_failed_checks = _get_expected_failed_checks(est)
    # This is to make sure we test a class that has some expected failures
    assert len(expected_failed_checks) > 0
    with warnings.catch_warnings(record=True):