Repository: scikit-learn/scikit-learn
File: sklearn/linear_model/tests/test_logistic.py (Python)

Code before:
        assert set(y[train]) & set(y[test]) == set()

    clf = LogisticRegressionCV(cv=cv, use_legacy_attributes=False).fit(X, y)
    # We expect accuracy to be exactly 0 because train and test sets have
    # non-overlapping labels
    assert np.all(clf.scores_ == 0.0)

    # Because of a LogisticRegressionCV bug, we need to create our own scoring
    # function to pass explicitly the labels
    scoring = make_scorer(
        brier_score_loss,
        greater_is_better=False,
        response_method="predict_proba",
        scale_by_half=True,
        labels=classes,
    )
    clf = LogisticRegressionCV(cv=cv, scoring=scoring, use_legacy_attributes=False).fit(
        X, y
    )
    brier_scores = -clf.scores_
    # We expect the scores to be bad because train and test sets have
    # non-overlapping labels
    assert np.all(brier_scores > 0.7)


def test_logistic_regression_solvers(global_random_seed):
    """Test solvers converge to the same result."""
    X, y = make_classification(
        n_samples=200, n_features=10, n_informative=5, random_state=global_random_seed
    )

    params = dict(C=0.1, fit_intercept=False, random_state=global_random_seed)

    classifiers = {
        solver: LogisticRegression(solver=solver, **params).fit(X, y)
        for solver in SOLVERS
    }

    for solver_1, solver_2 in itertools.combinations(classifiers, r=2):
        assert_allclose(
            classifiers[solver_1].coef_,
            classifiers[solver_2].coef_,
            atol=1e-3,
            rtol=1e-4,
            err_msg=f"Compare {solver_1} vs {solver_2}",
        )


# FIXME: the random state is fixed in the following test because SAG fails
# to converge to the same results as BFGS for 20% of the cases. Usually it
# means that there is one coefficient that is slightly different.

Review comment (suggestion):
I still kept a test with brier_score to try not to annoy you too much @lorentzenchr, I can also remove it whatever you prefer. 

Code after:
        assert set(y[train]) & set(y[test]) == set()

    clf = LogisticRegressionCV(cv=cv, use_legacy_attributes=False).fit(X, y)
    # We expect accuracy to be exactly 0 because train and test sets have
    # non-overlapping labels
    assert np.all(clf.scores_ == 0.0)

    # We use a proper scoring rule, i.e. the Brier score, to evaluate our classifier.
    # Because of a bug in LogisticRegressionCV, we need to create our own scoring
    # function to pass explicitly the labels.
    scoring = make_scorer(
        brier_score_loss,
        greater_is_better=False,
        response_method="predict_proba",
        scale_by_half=True,
        labels=classes,
    )
    # We set small Cs, that is strong penalty as the best C is likely the smallest one.
    clf = LogisticRegressionCV(
        cv=cv, scoring=scoring, Cs=np.logspace(-6, 3, 10), use_legacy_attributes=False
    ).fit(X, y)
    assert clf.C_ == 1e-6  # smallest value of provided Cs
    brier_scores = -clf.scores_
    # We expect the scores to be bad because train and test sets have
    # non-overlapping labels
    assert np.all(brier_scores > 0.7)
    # But the best score should be better than the worst value of 1.
    assert np.min(brier_scores) < 0.8


def test_logistic_regression_solvers(global_random_seed):
    """Test solvers converge to the same result."""
    X, y = make_classification(
        n_samples=200, n_features=10, n_informative=5, random_state=global_random_seed
    )

    params = dict(C=0.1, fit_intercept=False, random_state=global_random_seed)

    classifiers = {
        solver: LogisticRegression(solver=solver, **params).fit(X, y)
        for solver in SOLVERS
    }

    for solver_1, solver_2 in itertools.combinations(classifiers, r=2):
        assert_allclose(
            classifiers[solver_1].coef_,
            classifiers[solver_2].coef_,
            atol=1e-3,
            rtol=1e-4,
            err_msg=f"Compare {solver_1} vs {solver_2}",
        )