Repository: scikit-learn/scikit-learn

Issue:
# Negative Log Loss metric fails in LogisticRegressionCV when class is unobserved in CV fold

#### Description
When LogisticRegressionCV is called on a multi-class dataset with a "low-data" class, some CV folds are forced to drop the rare-class entirely.  This propagates through to the `neg-log-loss` metric as an exception:

#### Steps/Code to Reproduce

```python
"""
Modify the digits dataset to be very low in 9s, to trigger
an exception in the negative log loss metric.
"""

import sklearn.datasets
import sklearn.linear_model
import numpy as np

X, y = sklearn.datasets.load_digits(return_X_y=True)

n9 = 2

y9 = y[y == 9]
X9 = X[y == 9]

X = X[y != 9]
y = y[y != 9]

X = np.vstack([X, X9[0:n9]])
y = np.hstack([y, y9[0:n9]])

multi_class = "multinomial"
scoring = "neg_log_loss"
random_state = 1

model = sklearn.linear_model.LogisticRegressionCV(multi_class=multi_class, scoring=scoring, random_state=random_state)
model.fit(X, y)

```

#### Expected Results
I would expect this to run without an exception, or raise an exception before model training is attempted.   In particular, if you assume that 0 log(0) -> 0, this should "just work" IMHO.  

#### Actual Results

```
....env/lib/python3.6/site-packages/sklearn/linear_model/logistic.py in _log_reg_scoring_path(X, y, train, test, pos_class, Cs, scoring, fit_intercept, max_iter, tol, class_weight, verbose, solver, penalty, dual, intercept_scaling, multi_class, random_state, max_squared_sum, sample_weight, l1_ratio)
   1195             scores.append(log_reg.score(X_test, y_test))
   1196         else:
-> 1197             scores.append(scoring(log_reg, X_test, y_test))
   1198 
   1199     return coefs, Cs, np.array(scores), n_iter

....env/lib/python3.6/site-packages/sklearn/metrics/scorer.py in __call__(self, clf, X, y, sample_weight)
    138                                                  **self._kwargs)
    139         else:
--> 140             return self._sign * self._score_func(y, y_pred, **self._kwargs)
    141 
    142     def _factory_args(self):

....env/lib/python3.6/site-packages/sklearn/metrics/classification.py in log_loss(y_true, y_pred, eps, normalize, sample_weight, labels)
   2164                              "y_true: {2}".format(transformed_labels.shape[1],
   2165                                                   y_pred.shape[1],
-> 2166                                                   lb.classes_))
   2167         else:
   2168             raise ValueError('The number of classes in labels is different '

ValueError: y_true and y_pred contain different number of classes 9, 10. Please provide the true labels explicitly through the labels argument. Classes found in y_true: [0 1 2 3 4 5 6 7 8]
```

#### Versions

```
System:
    python: 3.6.7 | packaged by conda-forge | (default, Jul  2 2019, 02:07:37)  [GCC 4.2.1 Compatible Clang 4.0.1 (tags/RELEASE_401/final)]
executable: miniconda3/envs/eda/bin/python
   machine: Darwin-18.7.0-x86_64-i386-64bit

Python deps:
       pip: 19.3.1
setuptools: 41.4.0
   sklearn: 0.21.3
     numpy: 1.17.2
     scipy: 1.3.1
    Cython: None
    pandas: 0.25.2
```

One workaround that I've been able to implement is using `functools.partial` to create a custom loss function that takes the output of `predict_proba()` and expands it by filling zeros into the missing label.  This approach assumes that the missing label is always at the end of the array, which I think is reasonable but needs to be tested.  

Implementation patch:
diff --git a/sklearn/linear_model/_logistic.py b/sklearn/linear_model/_logistic.py
index 90875a32dca27..3a9104eb2c300 100644
--- a/sklearn/linear_model/_logistic.py
+++ b/sklearn/linear_model/_logistic.py
@@ -5,6 +5,7 @@
 # Authors: The scikit-learn developers
 # SPDX-License-Identifier: BSD-3-Clause
 
+import inspect
 import numbers
 import warnings
 from numbers import Integral, Real
@@ -27,7 +28,7 @@
 from sklearn.linear_model._glm.glm import NewtonCholeskySolver
 from sklearn.linear_model._linear_loss import LinearModelLoss
 from sklearn.linear_model._sag import sag_solver
-from sklearn.metrics import get_scorer, get_scorer_names
+from sklearn.metrics import get_scorer, get_scorer_names, make_scorer
 from sklearn.model_selection import check_cv
 from sklearn.preprocessing import LabelEncoder
 from sklearn.svm._base import _fit_liblinear
@@ -308,6 +309,7 @@ def _logistic_regression_path(
         w0 = np.zeros(
             n_features + int(fit_intercept), dtype=_matching_numpy_dtype(X, xp=xp)
         )
+        # classes[1] is the "positive label"
         mask = xp.asarray(y == classes[1], device=device_)
         y_bin = xp.ones(y.shape, dtype=X.dtype, device=device_)
         if solver == "liblinear":
@@ -749,8 +751,53 @@ def _log_reg_scoring_path(
 
     scores = list()
 
+    # Prepare the call to get the score per fold: calc_score
     scoring = get_scorer(scoring)
-    for w in coefs:
+    if scoring is None:
+
+        def calc_score(log_reg):
+            return log_reg.score(X_test, y_test, sample_weight=sw_test)
+
+    else:
+        is_binary = len(classes) <= 2
+        score_params = score_params or {}
+        score_params = _check_method_params(X=X, params=score_params, indices=test)
+        # We need to pass the classes as "labels" argument to scorers that support
+        # it, e.g. scoring = "neg_brier_score", because y_test may not contain all
+        # class labels.
+        # There are at least 2 possibilities:
+        # 1. Metadata routing is enabled: A try except clause is possible with
+        #   adding labels to score_params. We could then pass the already instantiated
+        #   log_reg instance to scoring.
+        # 2. We reconstruct the scorer and pass labels as kwargs explicitly.
+        # We implement the 2nd option even if it seems a bit hacky because it works
+        # with and without metadata routing.
+        if hasattr(scoring, "_score_func"):
+            sig = inspect.signature(scoring._score_func).parameters
+        else:
+            sig = []
+
+        if (is_binary and "labels" in sig and "pos_label" in sig) or (
+            len(classes) >= 3 and "labels" in sig
+        ):
+            pos_label_kwarg = {}
+            if is_binary:
+                # see _logistic_regression_path
+                pos_label_kwarg["pos_label"] = classes[-1]
+            scoring = make_scorer(
+                scoring._score_func,
+                greater_is_better=True if scoring._sign == 1 else False,
+                response_method=scoring._response_method,
+                labels=classes,
+                **pos_label_kwarg,
+                **getattr(scoring, "_kwargs", {}),
+            )
+
+        def calc_score(log_reg):
+            return scoring(log_reg, X_test, y_test, **score_params)
+
+    for w, C in zip(coefs, Cs):
+        log_reg.C = C
         if fit_intercept:
             log_reg.coef_ = w[..., :-1]
             log_reg.intercept_ = w[..., -1]
@@ -758,15 +805,8 @@ def _log_reg_scoring_path(
             log_reg.coef_ = w
             log_reg.intercept_ = 0.0
 
-        if scoring is None:
-            scores.append(log_reg.score(X_test, y_test, sample_weight=sw_test))
-        else:
-            score_params = score_params or {}
-            score_params = _check_method_params(X=X, params=score_params, indices=test)
-            # FIXME: If scoring = "neg_brier_score" and if not all class labels
-            # are present in y_test, the following fails. Maybe we can pass
-            # "labels=classes" to the call of scoring.
-            scores.append(scoring(log_reg, X_test, y_test, **score_params))
+        scores.append(calc_score(log_reg))
+
     return coefs, Cs, np.array(scores), n_iter
 
 


Test patch:
diff --git a/sklearn/linear_model/tests/test_logistic.py b/sklearn/linear_model/tests/test_logistic.py
index 928c11c61953c..da5818b1d7139 100644
--- a/sklearn/linear_model/tests/test_logistic.py
+++ b/sklearn/linear_model/tests/test_logistic.py
@@ -23,7 +23,7 @@
     _log_reg_scoring_path,
     _logistic_regression_path,
 )
-from sklearn.metrics import brier_score_loss, get_scorer, log_loss, make_scorer
+from sklearn.metrics import brier_score_loss, get_scorer, log_loss
 from sklearn.model_selection import (
     GridSearchCV,
     KFold,
@@ -729,26 +729,68 @@ def test_multinomial_cv_iris(use_legacy_attributes):
     assert np.all(clf.scores_ == 0.0)
 
     # We use a proper scoring rule, i.e. the Brier score, to evaluate our classifier.
-    # Because of a bug in LogisticRegressionCV, we need to create our own scoring
-    # function to pass explicitly the labels.
-    scoring = make_scorer(
-        brier_score_loss,
-        greater_is_better=False,
-        response_method="predict_proba",
-        scale_by_half=True,
-        labels=classes,
-    )
     # We set small Cs, that is strong penalty as the best C is likely the smallest one.
     clf = LogisticRegressionCV(
-        cv=cv, scoring=scoring, Cs=np.logspace(-6, 3, 10), use_legacy_attributes=False
+        cv=cv,
+        scoring="neg_brier_score",
+        Cs=np.logspace(-6, 3, 10),
+        use_legacy_attributes=False,
     ).fit(X, y)
     assert clf.C_ == 1e-6  # smallest value of provided Cs
     brier_scores = -clf.scores_
     # We expect the scores to be bad because train and test sets have
     # non-overlapping labels
-    assert np.all(brier_scores > 0.7)
+    assert np.all(brier_scores > 0.7 * 2)  # times 2 because scale_by_half=False
     # But the best score should be better than the worst value of 1.
-    assert np.min(brier_scores) < 0.8
+    assert np.min(brier_scores) < 0.8 * 2  # times 2 because scale_by_half=False
+
+
+@pytest.mark.parametrize("enable_metadata_routing", [False, True])
+@pytest.mark.parametrize("n_classes", [2, 3])
+def test_logistic_cv_folds_with_classes_missing(enable_metadata_routing, n_classes):
+    """Test that LogisticRegressionCV correctly computes scores even when classes are
+    missing on CV folds.
+    """
+    with config_context(enable_metadata_routing=enable_metadata_routing):
+        y = np.array(["a", "a", "b", "b", "c", "c"])[: 2 * n_classes]
+        X = np.arange(2 * n_classes)[:, None]
+
+        # Test CV folds have missing class labels.
+        cv = KFold(n_splits=n_classes)
+        # Check this assumption.
+        for train, test in cv.split(X, y):
+            assert len(np.unique(y[train])) == n_classes - 1
+            assert len(np.unique(y[test])) == 1
+            assert set(y[train]) & set(y[test]) == set()
+
+        clf = LogisticRegressionCV(
+            cv=cv,
+            scoring="neg_brier_score",
+            Cs=np.logspace(-6, 6, 5),
+            l1_ratios=(0,),
+            use_legacy_attributes=False,
+        ).fit(X, y)
+
+        assert clf.C_ == 1e-6  # smallest value of provided Cs
+        for i, (train, test) in enumerate(cv.split(X, y)):
+            # We need to construct the logistic regression model, clf2, as it was fit on
+            # a single training fold.
+            clf2 = LogisticRegression(C=clf.C_).fit(X, y)
+            clf2.coef_ = clf.coefs_paths_[i, 0, 0, :, :-1]
+            clf2.intercept_ = clf.coefs_paths_[i, 0, 0, :, -1]
+            if n_classes <= 2:
+                bs = brier_score_loss(
+                    y[test],
+                    clf2.predict_proba(X[test]),
+                    pos_label="b",
+                    labels=["a", "b"],
+                )
+            else:
+                bs = brier_score_loss(
+                    y[test], clf2.predict_proba(X[test]), labels=["a", "b", "c"]
+                )
+
+            assert_allclose(-clf.scores_[i, 0, 0], bs)
 
 
 def test_logistic_regression_solvers(global_random_seed):