Repository: probabl-ai/skore

Issue:
in CrossValidationReporter plots neg_RMSE is not negative
### Describe the bug

![Capture d’écran du 2024-12-23 15-50-21](https://github.com/user-attachments/assets/e61f0efa-cc1b-46b4-97c5-52cd67d85b3f)


### Steps/Code to Reproduce

```python

import skore 
project = skore.create("my_project", overwrite = True)
from sklearn.datasets import make_regression
X, y = make_regression(n_samples=500, n_features=7, noise=1, random_state=1)
from sklearn.ensemble import HistGradientBoostingRegressor
reg = HistGradientBoostingRegressor()
cv_reporter_reg = skore.CrossValidationReporter(reg, X, y, cv=10)
project.put("cv_regression", cv_reporter_reg)
```

### Expected Behavior

if the value if positive, the name shouldn't start with `neg`. 

### Actual Behavior

Neg.

### Environment

```shell
System:
    python: 3.12.8 | packaged by Anaconda, Inc. | (main, Dec 11 2024, 16:31:09) [GCC 11.2.0]
executable: /home/marie/anaconda3/envs/skore_test/bin/python
   machine: Linux-6.8.0-49-generic-x86_64-with-glibc2.35

Python dependencies:
        skore: 0.0.0+unknown
          pip: 24.2
   setuptools: 75.1.0
    diskcache: 5.6.3
      fastapi: 0.115.6
        numpy: 2.2.0
 plotly<6,>=5: None
      pyarrow: 18.1.0
         rich: 13.9.4
scikit-learn<1.7: None
        skops: 0.11.0
      uvicorn: 0.32.1
```


Discussion:
glemaitre: I added a non-regression test and remove the current test. I think that the other tests in the file should be modified in the same spirit by reusing the fixture as much as possible.

I don't like that we have so much mocking to test functionalities. I think this is fine to make some short computation.

Patch:
diff --git a/skore/src/skore/sklearn/cross_validation/cross_validation_helpers.py b/skore/src/skore/sklearn/cross_validation/cross_validation_helpers.py
index 8387ff8..a4af04f 100644
--- a/skore/src/skore/sklearn/cross_validation/cross_validation_helpers.py
+++ b/skore/src/skore/sklearn/cross_validation/cross_validation_helpers.py
@@ -26,14 +26,16 @@ def _get_scorers_to_add(estimator, y) -> list[str]:
     if ml_task == "regression":
         return {
             "r2": "r2",
-            "neg_root_mean_squared_error": metrics.make_scorer(
-                metrics.root_mean_squared_error
+            "root_mean_squared_error": metrics.make_scorer(
+                metrics.root_mean_squared_error, response_method="predict"
             ),
         }
     if ml_task == "binary-classification":
         return {
             "roc_auc": "roc_auc",
-            "brier_score_loss": metrics.make_scorer(metrics.brier_score_loss),
+            "brier_score_loss": metrics.make_scorer(
+                metrics.brier_score_loss, response_method="predict_proba"
+            ),
             "recall": "recall",
             "precision": "precision",
         }
@@ -43,7 +45,9 @@ def _get_scorers_to_add(estimator, y) -> list[str]:
                 "recall_weighted": "recall_weighted",
                 "precision_weighted": "precision_weighted",
                 "roc_auc_ovr_weighted": "roc_auc_ovr_weighted",
-                "log_loss": metrics.make_scorer(metrics.log_loss),
+                "log_loss": metrics.make_scorer(
+                    metrics.log_loss, response_method="predict_proba"
+                ),
             }
         return {
             "recall_weighted": "recall_weighted",


Test patch:
diff --git a/skore/tests/integration/sklearn/test_cross_validate.py b/skore/tests/integration/sklearn/test_cross_validate.py
index 5354bdc..8c89927 100644
--- a/skore/tests/integration/sklearn/test_cross_validate.py
+++ b/skore/tests/integration/sklearn/test_cross_validate.py
@@ -5,10 +5,13 @@ import sklearn.model_selection
 from sklearn import datasets, linear_model
 from sklearn.cluster import KMeans
 from sklearn.ensemble import RandomForestClassifier
+from sklearn.linear_model import LinearRegression, LogisticRegression
+from sklearn.model_selection import cross_validate
 from sklearn.multiclass import OneVsOneClassifier
 from sklearn.svm import SVC
 from skore import CrossValidationReporter
 from skore.item.cross_validation_item import CrossValidationItem
+from skore.sklearn.cross_validation.cross_validation_helpers import _get_scorers_to_add
 
 
 @pytest.fixture
@@ -155,23 +158,103 @@ class TestMLTask:
         assert all(len(v) == 5 for v in cv_results.values())
 
 
-def prepare_cv():
-    from sklearn import datasets, linear_model
+@pytest.fixture
+def binary_classifier():
+    X, y = datasets.make_classification(n_classes=2, random_state=42)
+    return LogisticRegression(), X, y
+
 
-    diabetes = datasets.load_diabetes()
-    X = diabetes.data[:150]
-    y = diabetes.target[:150]
-    lasso = linear_model.Lasso()
+@pytest.fixture
+def multiclass_classifier():
+    X, y = datasets.make_classification(
+        n_classes=3, n_clusters_per_class=1, random_state=42
+    )
+    return LogisticRegression(), X, y
 
-    return lasso, X, y
 
+@pytest.fixture
+def single_output_regression():
+    X, y = datasets.make_regression(n_targets=1, random_state=42)
+    return LinearRegression(), X, y
 
-def test_put_cross_validation_reporter(in_memory_project):
-    project = in_memory_project
 
-    lasso, X, y = prepare_cv()
-    reporter = CrossValidationReporter(lasso, X, y, cv=3)
+@pytest.fixture
+def multi_output_regression():
+    X, y = datasets.make_regression(n_targets=2, random_state=42)
+    return LinearRegression(), X, y
+
+
+@pytest.mark.parametrize(
+    "fixture_name",
+    [
+        "binary_classifier",
+        "multiclass_classifier",
+        "single_output_regression",
+        "multi_output_regression",
+    ],
+)
+def test_cross_validation_reporter(in_memory_project, fixture_name, request):
+    """Check that we can serialize the `CrossValidationReporter` and retrieve it."""
+    model, X, y = request.getfixturevalue(fixture_name)
+    reporter = CrossValidationReporter(model, X, y, cv=3)
+
+    in_memory_project.put("cross-validation", reporter)
+
+    retrieved_item = in_memory_project.get_item("cross-validation")
+    assert isinstance(retrieved_item, CrossValidationItem)
+
+
+@pytest.mark.parametrize(
+    "fixture_name",
+    [
+        "binary_classifier",
+        "multiclass_classifier",
+        "single_output_regression",
+        "multi_output_regression",
+    ],
+)
+def test_cross_validation_reporter_equivalence_cross_validate(
+    in_memory_project, fixture_name, request
+):
+    """Check that we have an equivalent result to `cross_validate`."""
+    # mapping between the scorers names in skore and in sklearn
+    map_skore_to_sklearn = {
+        "r2": "r2",
+        "root_mean_squared_error": "neg_root_mean_squared_error",
+        "roc_auc": "roc_auc",
+        "brier_score_loss": "neg_brier_score",
+        "recall": "recall",
+        "precision": "precision",
+        "recall_weighted": "recall_weighted",
+        "precision_weighted": "precision_weighted",
+        "roc_auc_ovr_weighted": "roc_auc_ovr_weighted",
+        "log_loss": "neg_log_loss",
+    }
+    model, X, y = request.getfixturevalue(fixture_name)
+    reporter = CrossValidationReporter(
+        model, X, y, cv=3, return_estimator=True, return_train_score=True
+    )
 
-    project.put("cross-validation", reporter)
+    scorers_used_skore = _get_scorers_to_add(model, y)
+    scorers_sklearn = [map_skore_to_sklearn[k] for k in scorers_used_skore]
+    cv_results_sklearn = cross_validate(
+        model,
+        X,
+        y,
+        cv=3,
+        scoring=scorers_sklearn,
+        return_estimator=True,
+        return_train_score=True,
+    )
 
-    assert isinstance(project.get_item("cross-validation"), CrossValidationItem)
+    # check the equivalence between the scores
+    for scorer_skore_name in scorers_used_skore:
+        for type_set in ["test", "train"]:
+            score_skore = reporter._cv_results[f"{type_set}_{scorer_skore_name}"]
+            score_sklearn = cv_results_sklearn[
+                f"{type_set}_{map_skore_to_sklearn[scorer_skore_name]}"
+            ]
+            if map_skore_to_sklearn[scorer_skore_name].startswith("neg_"):
+                numpy.testing.assert_allclose(score_skore, -score_sklearn)
+            else:
+                numpy.testing.assert_allclose(score_skore, score_sklearn)