Repository: probabl-ai/skore

Issue:
api: return Python native type when calling metrics function
As discussed in https://github.com/probabl-ai/skore/issues/1259, we should be returning the most appropriate Python type (float or dict) when calling a function delegating the concatenation possibility to the `.report_metrics` methods.

Discussion:
github-actions[bot]: [Documentation preview](https://skoredoc.k.probabl.dev/1283) @ b3e8a9aa4593ea6f6bec10316a060f07d9465e6c
<!-- Sticky Pull Request Commentdocumentation-preview -->
glemaitre: It is ready for review @thomass-dev @auguste-probabl @sylvaincom 

I see that we should most probably drop the `plot_estimator_report.py` soonish since we split it in use case.

Patch:
diff --git a/examples/model_evaluation/plot_estimator_report.py b/examples/model_evaluation/plot_estimator_report.py
index 340190c..22c4f34 100644
--- a/examples/model_evaluation/plot_estimator_report.py
+++ b/examples/model_evaluation/plot_estimator_report.py
@@ -13,8 +13,9 @@ quickly get insights from any scikit-learn estimator.
 # Loading our dataset and defining our estimator
 # ==============================================
 #
-# First, we load a dataset from skrub. Our goal is to predict if a company paid a physician. The ultimate goal is to
-# detect potential conflict of interest when it comes to the actual problem that we want to solve.
+# First, we load a dataset from skrub. Our goal is to predict if a company paid a
+# physician. The ultimate goal is to detect potential conflict of interest when it comes
+# to the actual problem that we want to solve.
 
 # %%
 from skrub.datasets import fetch_open_payments
@@ -79,15 +80,12 @@ from skore import EstimatorReport
 report = EstimatorReport(
     estimator, X_train=X_train, y_train=y_train, X_test=X_test, y_test=y_test
 )
-report
 
 # %%
 #
 # Once the report is created, we get some information regarding the available tools
-# allowing us to get some insights from our specific model on our specific task.
-#
-# We can get a similar information if we call the :meth:`~skore.EstimatorReport.help`
-# method.
+# allowing us to get some insights from our specific model on our specific task by
+# calling the :meth:`~skore.EstimatorReport.help` method.
 report.help()
 
 # %%
@@ -257,10 +255,7 @@ report.metrics.accuracy()
 # We can now compute the cost of our operational decision.
 start = time.time()
 cost = report.metrics.custom_metric(
-    metric_function=operational_decision_cost,
-    metric_name="Operational Decision Cost",
-    response_method="predict",
-    amount=amount,
+    metric_function=operational_decision_cost, response_method="predict", amount=amount
 )
 end = time.time()
 cost
@@ -276,10 +271,7 @@ report.clear_cache()
 # %%
 start = time.time()
 cost = report.metrics.custom_metric(
-    metric_function=operational_decision_cost,
-    metric_name="Operational Decision Cost",
-    response_method="predict",
-    amount=amount,
+    metric_function=operational_decision_cost, response_method="predict", amount=amount
 )
 end = time.time()
 cost
@@ -294,12 +286,9 @@ print(f"Time taken to compute the cost: {end - start:.2f} seconds")
 # the predictions.
 report.metrics.report_metrics(
     scoring=["precision", "recall", operational_decision_cost],
+    scoring_names=["Precision", "Recall", "Operational Decision Cost"],
     pos_label=pos_label,
-    scoring_kwargs={
-        "amount": amount,
-        "response_method": "predict",
-        "metric_name": "Operational Decision Cost",
-    },
+    scoring_kwargs={"amount": amount, "response_method": "predict"},
 )
 
 # %%
@@ -310,16 +299,14 @@ report.metrics.report_metrics(
 # function.
 from sklearn.metrics import make_scorer, f1_score
 
-f1_scorer = make_scorer(
-    f1_score, response_method="predict", metric_name="F1 Score", pos_label=pos_label
-)
+f1_scorer = make_scorer(f1_score, response_method="predict", pos_label=pos_label)
 operational_decision_cost_scorer = make_scorer(
-    operational_decision_cost,
-    response_method="predict",
-    metric_name="Operational Decision Cost",
-    amount=amount,
+    operational_decision_cost, response_method="predict", amount=amount
+)
+report.metrics.report_metrics(
+    scoring=[f1_scorer, operational_decision_cost_scorer],
+    scoring_names=["F1 Score", "Operational Decision Cost"],
 )
-report.metrics.report_metrics(scoring=[f1_scorer, operational_decision_cost_scorer])
 
 # %%
 #
diff --git a/skore/src/skore/persistence/view/view.py b/skore/src/skore/persistence/view/view.py
index e9a4592..2f4a013 100644
--- a/skore/src/skore/persistence/view/view.py
+++ b/skore/src/skore/persistence/view/view.py
@@ -12,6 +12,7 @@ class View:
 
     Examples
     --------
+    >>> from skore.persistence.view import View
     >>> View(layout=["a", "b"])
     View(...)
     """
diff --git a/skore/src/skore/sklearn/_cross_validation/metrics_accessor.py b/skore/src/skore/sklearn/_cross_validation/metrics_accessor.py
index c7e168d..0257e31 100644
--- a/skore/src/skore/sklearn/_cross_validation/metrics_accessor.py
+++ b/skore/src/skore/sklearn/_cross_validation/metrics_accessor.py
@@ -1,6 +1,7 @@
 import joblib
 import numpy as np
 import pandas as pd
+from sklearn.metrics import make_scorer
 from sklearn.utils.metaestimators import available_if
 
 from skore.externals._pandas_accessors import DirNamesMixin
@@ -20,17 +21,17 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
     You can access this accessor using the `metrics` attribute.
     """
 
-    _SCORE_OR_LOSS_ICONS = {
-        "accuracy": "(↗︎)",
-        "precision": "(↗︎)",
-        "recall": "(↗︎)",
-        "brier_score": "(↘︎)",
-        "roc_auc": "(↗︎)",
-        "log_loss": "(↘︎)",
-        "r2": "(↗︎)",
-        "rmse": "(↘︎)",
-        "report_metrics": "",
-        "custom_metric": "",
+    _SCORE_OR_LOSS_INFO = {
+        "accuracy": {"name": "Accuracy", "icon": "(↗︎)"},
+        "precision": {"name": "Precision", "icon": "(↗︎)"},
+        "recall": {"name": "Recall", "icon": "(↗︎)"},
+        "brier_score": {"name": "Brier score", "icon": "(↘︎)"},
+        "roc_auc": {"name": "ROC AUC", "icon": "(↗︎)"},
+        "log_loss": {"name": "Log loss", "icon": "(↘︎)"},
+        "r2": {"name": "R²", "icon": "(↗︎)"},
+        "rmse": {"name": "RMSE", "icon": "(↘︎)"},
+        "custom_metric": {"name": "Custom metric", "icon": ""},
+        "report_metrics": {"name": "Report metrics", "icon": ""},
     }
 
     def __init__(self, parent):
@@ -210,8 +211,8 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
         LogisticRegression Split #0       0.94...
                            Split #1       0.94...
         """
-        return self._compute_metric_scores(
-            report_metric_name="accuracy",
+        return self.report_metrics(
+            scoring=["accuracy"],
             data_source=data_source,
             aggregate=aggregate,
         )
@@ -285,16 +286,16 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
         >>> report = CrossValidationReport(classifier, X=X, y=y, cv_splitter=2)
         >>> report.metrics.precision()
         Metric                      Precision (↗︎)
-        Class label                              0         1
+        Label / Average                          0         1
         LogisticRegression Split #0       0.96...   0.93...
                            Split #1       0.90...   0.96...
         """
-        return self._compute_metric_scores(
-            report_metric_name="precision",
+        return self.report_metrics(
+            scoring=["precision"],
             data_source=data_source,
             aggregate=aggregate,
-            average=average,
             pos_label=pos_label,
+            scoring_kwargs={"average": average},
         )
 
     @available_if(
@@ -367,16 +368,16 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
         >>> report = CrossValidationReport(classifier, X=X, y=y, cv_splitter=2)
         >>> report.metrics.recall()
         Metric                      Recall (↗︎)
-        Class label                           0        1
+        Label / Average                          0         1
         LogisticRegression Split #0    0.87...   0.98...
                            Split #1    0.94...   0.94...
         """
-        return self._compute_metric_scores(
-            report_metric_name="recall",
+        return self.report_metrics(
+            scoring=["recall"],
             data_source=data_source,
             aggregate=aggregate,
-            average=average,
             pos_label=pos_label,
+            scoring_kwargs={"average": average},
         )
 
     @available_if(
@@ -414,8 +415,8 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
         LogisticRegression Split #0          0.04...
                            Split #1          0.04...
         """
-        return self._compute_metric_scores(
-            report_metric_name="brier_score",
+        return self.report_metrics(
+            scoring=["brier_score"],
             data_source=data_source,
             aggregate=aggregate,
         )
@@ -443,8 +444,7 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
             - "test" : use the test set provided when creating the report.
             - "train" : use the train set provided when creating the report.
 
-        average : {"auto", "macro", "micro", "weighted", "samples"}, \
-                default=None
+        average : {"macro", "micro", "weighted", "samples"}, default=None
             Average to compute the ROC AUC score in a multiclass setting. By default,
             no average is computed. Otherwise, this determines the type of averaging
             performed on the data.
@@ -498,12 +498,11 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
         LogisticRegression Split #0      0.99...
                            Split #1      0.98...
         """
-        return self._compute_metric_scores(
-            report_metric_name="roc_auc",
+        return self.report_metrics(
+            scoring=["roc_auc"],
             data_source=data_source,
             aggregate=aggregate,
-            average=average,
-            multi_class=multi_class,
+            scoring_kwargs={"average": average, "multi_class": multi_class},
         )
 
     @available_if(
@@ -543,8 +542,8 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
         LogisticRegression Split #0       0.1...
                            Split #1       0.1...
         """
-        return self._compute_metric_scores(
-            report_metric_name="log_loss",
+        return self.report_metrics(
+            scoring=["log_loss"],
             data_source=data_source,
             aggregate=aggregate,
         )
@@ -598,11 +597,11 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
         Ridge Split #0  0.36...
               Split #1  0.39...
         """
-        return self._compute_metric_scores(
-            report_metric_name="r2",
+        return self.report_metrics(
+            scoring=["r2"],
             data_source=data_source,
             aggregate=aggregate,
-            multioutput=multioutput,
+            scoring_kwargs={"multioutput": multioutput},
         )
 
     @available_if(_check_supported_ml_task(supported_ml_tasks=["regression"]))
@@ -654,11 +653,11 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
         Ridge Split #0  59.9...
               Split #1  61.4...
         """
-        return self._compute_metric_scores(
-            report_metric_name="rmse",
+        return self.report_metrics(
+            scoring=["rmse"],
             data_source=data_source,
             aggregate=aggregate,
-            multioutput=multioutput,
+            scoring_kwargs={"multioutput": multioutput},
         )
 
     def custom_metric(
@@ -732,15 +731,20 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
         Ridge Split #0  50.1...
               Split #1  52.6...
         """
-        return self._compute_metric_scores(
-            report_metric_name="custom_metric",
-            data_source=data_source,
-            aggregate=aggregate,
-            metric_function=metric_function,
+        # create a scorer with `greater_is_better=True` to not alter the output of
+        # `metric_function`
+        scorer = make_scorer(
+            metric_function,
+            greater_is_better=True,
             response_method=response_method,
-            metric_name=metric_name,
             **kwargs,
         )
+        return self.report_metrics(
+            scoring=[scorer],
+            data_source=data_source,
+            aggregate=aggregate,
+            scoring_names=[metric_name],
+        )
 
     ####################################################################################
     # Methods related to the help tree
@@ -768,15 +772,16 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
         """Override format method for metrics-specific naming."""
         method_name = f"{name}(...)"
         method_name = method_name.ljust(22)
-        if name in self._SCORE_OR_LOSS_ICONS and self._SCORE_OR_LOSS_ICONS[name] in (
-            "(↗︎)",
-            "(↘︎)",
-        ):
-            if self._SCORE_OR_LOSS_ICONS[name] == "(↗︎)":
-                method_name += f"[cyan]{self._SCORE_OR_LOSS_ICONS[name]}[/cyan]"
+        if name in self._SCORE_OR_LOSS_INFO and self._SCORE_OR_LOSS_INFO[name][
+            "icon"
+        ] in ("(↗︎)", "(↘︎)"):
+            if self._SCORE_OR_LOSS_INFO[name]["icon"] == "(↗︎)":
+                method_name += f"[cyan]{self._SCORE_OR_LOSS_INFO[name]['name']}[/cyan]"
                 return method_name.ljust(43)
             else:  # (↘︎)
-                method_name += f"[orange1]{self._SCORE_OR_LOSS_ICONS[name]}[/orange1]"
+                method_name += (
+                    f"[orange1]{self._SCORE_OR_LOSS_INFO[name]['name']}[/orange1]"
+                )
                 return method_name.ljust(49)
         else:
             return method_name.ljust(29)
diff --git a/skore/src/skore/sklearn/_estimator/metrics_accessor.py b/skore/src/skore/sklearn/_estimator/metrics_accessor.py
index 48d876a..d460385 100644
--- a/skore/src/skore/sklearn/_estimator/metrics_accessor.py
+++ b/skore/src/skore/sklearn/_estimator/metrics_accessor.py
@@ -24,23 +24,22 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
     You can access this accessor using the `metrics` attribute.
     """
 
-    _SCORE_OR_LOSS_ICONS = {
-        "accuracy": "(↗︎)",
-        "precision": "(↗︎)",
-        "recall": "(↗︎)",
-        "brier_score": "(↘︎)",
-        "roc_auc": "(↗︎)",
-        "log_loss": "(↘︎)",
-        "r2": "(↗︎)",
-        "rmse": "(↘︎)",
-        "report_metrics": "",
-        "custom_metric": "",
+    _SCORE_OR_LOSS_INFO = {
+        "accuracy": {"name": "Accuracy", "icon": "(↗︎)"},
+        "precision": {"name": "Precision", "icon": "(↗︎)"},
+        "recall": {"name": "Recall", "icon": "(↗︎)"},
+        "brier_score": {"name": "Brier score", "icon": "(↘︎)"},
+        "roc_auc": {"name": "ROC AUC", "icon": "(↗︎)"},
+        "log_loss": {"name": "Log loss", "icon": "(↘︎)"},
+        "r2": {"name": "R²", "icon": "(↗︎)"},
+        "rmse": {"name": "RMSE", "icon": "(↘︎)"},
+        "custom_metric": {"name": "Custom metric", "icon": ""},
+        "report_metrics": {"name": "Report metrics", "icon": ""},
     }
 
     def __init__(self, parent):
         super().__init__(parent)
 
-    # TODO: should build on the `add_scorers` function
     def report_metrics(
         self,
         *,
@@ -185,24 +184,31 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
                             )
                     elif pos_label is not None:
                         metrics_kwargs["pos_label"] = pos_label
+                if metric_name is None:
+                    metric_name = metric._score_func.__name__
             elif isinstance(metric, str) or callable(metric):
                 if isinstance(metric, str):
                     err_msg = (
                         f"Invalid metric: {metric!r}. Please use a valid metric"
                         " from the list of supported metrics: "
-                        f"{list(self._SCORE_OR_LOSS_ICONS.keys())}"
+                        f"{list(self._SCORE_OR_LOSS_INFO.keys())}"
                     )
                     if (
                         metric.startswith("_")
-                        and metric[1:] not in self._SCORE_OR_LOSS_ICONS
+                        and metric[1:] not in self._SCORE_OR_LOSS_INFO
                     ):
                         raise ValueError(err_msg)
                     if not metric.startswith("_"):
-                        if metric not in self._SCORE_OR_LOSS_ICONS:
+                        if metric not in self._SCORE_OR_LOSS_INFO:
                             raise ValueError(err_msg)
                         metric = f"_{metric}"
                     metric_fn = getattr(self, metric)
                     metrics_kwargs = {"data_source_hash": data_source_hash}
+                    if metric_name is None:
+                        metric_name = (
+                            f"{self._SCORE_OR_LOSS_INFO[metric[1:]]['name']} "
+                            f"{self._SCORE_OR_LOSS_INFO[metric[1:]]['icon']}"
+                        )
                 else:
                     metric_fn = partial(self._custom_metric, metric_function=metric)
                     if scoring_kwargs is None:
@@ -217,6 +223,8 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
                             if param in scoring_kwargs
                         }
                     metrics_kwargs["data_source_hash"] = data_source_hash
+                    if metric_name is None:
+                        metric_name = metric.__name__
                 metrics_params = inspect.signature(metric_fn).parameters
                 if scoring_kwargs is not None:
                     for param in metrics_params:
@@ -229,14 +237,69 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
                     f"Invalid type of metric: {type(metric)} for {metric!r}"
                 )
 
-            if metric_name is not None:
-                metrics_kwargs["metric_name"] = metric_name
+            score = metric_fn(data_source=data_source, X=X, y=y, **metrics_kwargs)
+
+            if self._parent._ml_task == "binary-classification":
+                if isinstance(score, dict):
+                    classes = list(score.keys())
+                    columns = pd.MultiIndex.from_arrays(
+                        [[metric_name] * len(classes), classes],
+                        names=["Metric", "Label / Average"],
+                    )
+                    score = np.hstack([score[c] for c in classes]).reshape(1, -1)
+                elif "average" in metrics_kwargs:
+                    if metrics_kwargs["average"] == "binary":
+                        columns = pd.MultiIndex.from_arrays(
+                            [[metric_name], [pos_label]],
+                            names=["Metric", "Label / Average"],
+                        )
+                    elif metrics_kwargs["average"] is not None:
+                        columns = pd.MultiIndex.from_arrays(
+                            [[metric_name], [metrics_kwargs["average"]]],
+                            names=["Metric", "Label / Average"],
+                        )
+                    else:
+                        columns = pd.Index([metric_name], name="Metric")
+                    score = np.array(score).reshape(1, -1)
+                else:
+                    columns = pd.Index([metric_name], name="Metric")
+                    score = np.array(score).reshape(1, -1)
+            elif self._parent._ml_task == "multiclass-classification":
+                if isinstance(score, dict):
+                    classes = list(score.keys())
+                    columns = pd.MultiIndex.from_arrays(
+                        [[metric_name] * len(classes), classes],
+                        names=["Metric", "Label / Average"],
+                    )
+                    score = np.hstack([score[c] for c in classes]).reshape(1, -1)
+                elif (
+                    "average" in metrics_kwargs
+                    and metrics_kwargs["average"] is not None
+                ):
+                    columns = pd.MultiIndex.from_arrays(
+                        [[metric_name], [metrics_kwargs["average"]]],
+                        names=["Metric", "Label / Average"],
+                    )
+                    score = np.array(score).reshape(1, -1)
+                else:
+                    columns = pd.Index([metric_name], name="Metric")
+                    score = np.array(score).reshape(1, -1)
+            elif self._parent._ml_task == "regression":
+                if isinstance(score, np.ndarray):
+                    columns = pd.MultiIndex.from_arrays(
+                        [[metric_name] * len(score), list(range(len(score)))],
+                        names=["Metric", "Output"],
+                    )
+                    score = score.reshape(1, -1)
+                else:
+                    columns = pd.Index([metric_name], name="Metric")
+                    score = np.array(score).reshape(1, -1)
+            else:  # unknown task - try our best
+                columns = [metric_name] if len(score) == 1 else None
+
             scores.append(
-                metric_fn(
-                    data_source=data_source,
-                    X=X,
-                    y=y,
-                    **metrics_kwargs,
+                pd.DataFrame(
+                    score, columns=columns, index=[self._parent.estimator_name_]
                 )
             )
 
@@ -251,11 +314,11 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
                 if hasattr(score, "columns") and not isinstance(
                     score.columns, pd.MultiIndex
                 ):
-                    name_index = (
-                        ["Metric", "Output"]
-                        if self._parent._ml_task == "regression"
-                        else ["Metric", "Class label"]
-                    )
+                    if self._parent._ml_task == "regression":
+                        name_index = ["Metric", "Output"]
+                    else:
+                        name_index = ["Metric", "Label / Average"]
+
                     scores[i].columns = pd.MultiIndex.from_tuples(
                         [(col, "") for col in score.columns],
                         names=name_index,
@@ -273,7 +336,6 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
         data_source_hash=None,
         response_method,
         pos_label=None,
-        metric_name=None,
         **metric_kwargs,
     ):
         if data_source_hash is None:
@@ -321,41 +383,20 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
             )
 
             score = metric_fn(y_true, y_pred, **kwargs)
+            if isinstance(score, np.floating):
+                # convert np.float64 and np.float32 to float for consistency across
+                # functions
+                score = float(score)
             self._parent._cache[cache_key] = score
 
-        score = np.array([score]) if not isinstance(score, np.ndarray) else score
-        metric_name = metric_name or metric_fn.__name__
-
-        if self._parent._ml_task in [
+        if self._parent._ml_task in (
             "binary-classification",
             "multiclass-classification",
-        ]:
-            if len(score) == 1:
-                columns = pd.Index([metric_name], name="Metric")
-            else:
-                classes = self._parent._estimator.classes_
-                columns = pd.MultiIndex.from_arrays(
-                    [[metric_name] * len(classes), classes],
-                    names=["Metric", "Class label"],
-                )
-                score = score.reshape(1, -1)
-        elif self._parent._ml_task == "regression":
-            if len(score) == 1:
-                columns = pd.Index([metric_name], name="Metric")
-            else:
-                columns = pd.MultiIndex.from_arrays(
-                    [
-                        [metric_name] * len(score),
-                        [f"#{i}" for i in range(len(score))],
-                    ],
-                    names=["Metric", "Output"],
-                )
-                score = score.reshape(1, -1)
-        else:  # unknown task - try our best
-            columns = [metric_name] if score.shape[0] == 1 else None
-        return pd.DataFrame(
-            score, columns=columns, index=[self._parent.estimator_name_]
-        )
+        ) and isinstance(score, np.ndarray):
+            return dict(zip(self._parent._estimator.classes_, score))
+        elif isinstance(score, np.ndarray):
+            return score[0] if score.size == 1 else score
+        return score
 
     @available_if(
         _check_supported_ml_task(
@@ -384,7 +425,7 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
 
         Returns
         -------
-        pd.DataFrame
+        float
             The accuracy score.
 
         Examples
@@ -405,8 +446,7 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
         ...     y_test=y_test,
         ... )
         >>> report.metrics.accuracy()
-        Metric              Accuracy (↗︎)
-        LogisticRegression       0.95...
+        0.95...
         """
         return self._accuracy(data_source=data_source, data_source_hash=None, X=X, y=y)
 
@@ -417,19 +457,13 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
         data_source_hash=None,
         X=None,
         y=None,
-        metric_name=None,
     ):
         """Private interface of `accuracy` to be able to pass `data_source_hash`.
 
         `data_source_hash` is either an `int` when we already computed the hash
         and are able to pass it around or `None` and thus trigger its computation
         in the underlying process.
-
-        `metric_name` allows to overwrite the default name of the metric in the report.
         """
-        if metric_name is None:
-            metric_name = f"Accuracy {self._SCORE_OR_LOSS_ICONS['accuracy']}"
-
         return self._compute_metric_scores(
             metrics.accuracy_score,
             X=X,
@@ -437,7 +471,6 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
             data_source=data_source,
             data_source_hash=data_source_hash,
             response_method="predict",
-            metric_name=metric_name,
         )
 
     @available_if(
@@ -497,7 +530,7 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
 
         Returns
         -------
-        pd.DataFrame
+        float or dict
             The precision score.
 
         Examples
@@ -518,8 +551,7 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
         ...     y_test=y_test,
         ... )
         >>> report.metrics.precision(pos_label=1)
-        Metric              Precision (↗︎)
-        LogisticRegression        0.98...
+        0.98...
         """
         return self._precision(
             data_source=data_source,
@@ -537,7 +569,6 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
         data_source_hash=None,
         X=None,
         y=None,
-        metric_name=None,
         average=None,
         pos_label=None,
     ):
@@ -546,17 +577,12 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
         `data_source_hash` is either an `int` when we already computed the hash
         and are able to pass it around or `None` and thus trigger its computation
         in the underlying process.
-
-        `metric_name` allows to overwrite the default name of the metric in the report.
         """
         if self._parent._ml_task == "binary-classification" and pos_label is not None:
             # if `pos_label` is specified by our user, then we can safely report only
             # the statistics of the positive class
             average = "binary"
 
-        if metric_name is None:
-            metric_name = f"Precision {self._SCORE_OR_LOSS_ICONS['precision']}"
-
         return self._compute_metric_scores(
             metrics.precision_score,
             X=X,
@@ -565,7 +591,6 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
             data_source_hash=data_source_hash,
             response_method="predict",
             pos_label=pos_label,
-            metric_name=metric_name,
             average=average,
         )
 
@@ -627,7 +652,7 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
 
         Returns
         -------
-        pd.DataFrame
+        float or dict
             The recall score.
 
         Examples
@@ -648,8 +673,7 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
         ...     y_test=y_test,
         ... )
         >>> report.metrics.recall(pos_label=1)
-        Metric              Recall (↗︎)
-        LogisticRegression     0.93...
+        0.93...
         """
         return self._recall(
             data_source=data_source,
@@ -667,7 +691,6 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
         data_source_hash=None,
         X=None,
         y=None,
-        metric_name=None,
         average=None,
         pos_label=None,
     ):
@@ -676,17 +699,12 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
         `data_source_hash` is either an `int` when we already computed the hash
         and are able to pass it around or `None` and thus trigger its computation
         in the underlying process.
-
-        `metric_name` allows to overwrite the default name of the metric in the report.
         """
         if self._parent._ml_task == "binary-classification" and pos_label is not None:
             # if `pos_label` is specified by our user, then we can safely report only
             # the statistics of the positive class
             average = "binary"
 
-        if metric_name is None:
-            metric_name = f"Recall {self._SCORE_OR_LOSS_ICONS['recall']}"
-
         return self._compute_metric_scores(
             metrics.recall_score,
             X=X,
@@ -695,7 +713,6 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
             data_source_hash=data_source_hash,
             response_method="predict",
             pos_label=pos_label,
-            metric_name=metric_name,
             average=average,
         )
 
@@ -724,7 +741,7 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
 
         Returns
         -------
-        pd.DataFrame
+        float
             The Brier score.
 
         Examples
@@ -745,8 +762,7 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
         ...     y_test=y_test,
         ... )
         >>> report.metrics.brier_score()
-        Metric              Brier score (↘︎)
-        LogisticRegression          0.03...
+        0.03...
         """
         return self._brier_score(
             data_source=data_source,
@@ -762,23 +778,17 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
         data_source_hash=None,
         X=None,
         y=None,
-        metric_name=None,
     ):
         """Private interface of `brier_score` to be able to pass `data_source_hash`.
 
         `data_source_hash` is either an `int` when we already computed the hash
         and are able to pass it around or `None` and thus trigger its computation
         in the underlying process.
-
-        `metric_name` allows to overwrite the default name of the metric in the report.
         """
         # The Brier score in scikit-learn request `pos_label` to ensure that the
         # integral encoding of `y_true` corresponds to the probabilities of the
         # `pos_label`. Since we get the predictions with `get_response_method`, we
         # can pass any `pos_label`, they will lead to the same result.
-        if metric_name is None:
-            metric_name = f"Brier score {self._SCORE_OR_LOSS_ICONS['brier_score']}"
-
         return self._compute_metric_scores(
             metrics.brier_score_loss,
             X=X,
@@ -786,7 +796,6 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
             data_source=data_source,
             data_source_hash=data_source_hash,
             response_method="predict_proba",
-            metric_name=metric_name,
             pos_label=self._parent._estimator.classes_[-1],
         )
 
@@ -817,8 +826,7 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
             New target on which to compute the metric. By default, we use the target
             provided when creating the report.
 
-        average : {"auto", "macro", "micro", "weighted", "samples"}, \
-                default=None
+        average : {"macro", "micro", "weighted", "samples"}, default=None
             Average to compute the ROC AUC score in a multiclass setting. By default,
             no average is computed. Otherwise, this determines the type of averaging
             performed on the data.
@@ -853,7 +861,7 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
 
         Returns
         -------
-        pd.DataFrame
+        float or dict
             The ROC AUC score.
 
         Examples
@@ -874,8 +882,7 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
         ...     y_test=y_test,
         ... )
         >>> report.metrics.roc_auc()
-        Metric              ROC AUC (↗︎)
-        LogisticRegression      0.99...
+        0.99...
         """
         return self._roc_auc(
             data_source=data_source,
@@ -893,7 +900,6 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
         data_source_hash=None,
         X=None,
         y=None,
-        metric_name=None,
         average=None,
         multi_class="ovr",
     ):
@@ -902,12 +908,7 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
         `data_source_hash` is either an `int` when we already computed the hash
         and are able to pass it around or `None` and thus trigger its computation
         in the underlying process.
-
-        `metric_name` allows to overwrite the default name of the metric in the report.
         """
-        if metric_name is None:
-            metric_name = f"ROC AUC {self._SCORE_OR_LOSS_ICONS['roc_auc']}"
-
         return self._compute_metric_scores(
             metrics.roc_auc_score,
             X=X,
@@ -915,7 +916,6 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
             data_source=data_source,
             data_source_hash=data_source_hash,
             response_method=["predict_proba", "decision_function"],
-            metric_name=metric_name,
             average=average,
             multi_class=multi_class,
         )
@@ -940,7 +940,7 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
 
         Returns
         -------
-        pd.DataFrame
+        float
             The log-loss.
 
         Examples
@@ -961,8 +961,7 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
         ...     y_test=y_test,
         ... )
         >>> report.metrics.log_loss()
-        Metric              Log loss (↘︎)
-        LogisticRegression       0.10...
+        0.10...
         """
         return self._log_loss(
             data_source=data_source,
@@ -978,19 +977,13 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
         data_source_hash=None,
         X=None,
         y=None,
-        metric_name=None,
     ):
         """Private interface of `log_loss` to be able to pass `data_source_hash`.
 
         `data_source_hash` is either an `int` when we already computed the hash
         and are able to pass it around or `None` and thus trigger its computation
         in the underlying process.
-
-        `metric_name` allows to overwrite the default name of the metric in the report.
         """
-        if metric_name is None:
-            metric_name = f"Log loss {self._SCORE_OR_LOSS_ICONS['log_loss']}"
-
         return self._compute_metric_scores(
             metrics.log_loss,
             X=X,
@@ -998,7 +991,6 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
             data_source=data_source,
             data_source_hash=data_source_hash,
             response_method="predict_proba",
-            metric_name=metric_name,
         )
 
     @available_if(_check_supported_ml_task(supported_ml_tasks=["regression"]))
@@ -1034,7 +1026,7 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
 
         Returns
         -------
-        pd.DataFrame
+        float or ndarray of shape (n_outputs,)
             The R² score.
 
         Examples
@@ -1055,8 +1047,7 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
         ...     y_test=y_test,
         ... )
         >>> report.metrics.r2()
-        Metric  R² (↗︎)
-        Ridge   0.35...
+        np.float64(0.35...)
         """
         return self._r2(
             data_source=data_source,
@@ -1073,7 +1064,6 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
         data_source_hash=None,
         X=None,
         y=None,
-        metric_name=None,
         multioutput="raw_values",
     ):
         """Private interface of `r2` to be able to pass `data_source_hash`.
@@ -1081,12 +1071,7 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
         `data_source_hash` is either an `int` when we already computed the hash
         and are able to pass it around or `None` and thus trigger its computation
         in the underlying process.
-
-        `metric_name` allows to overwrite the default name of the metric in the report.
         """
-        if metric_name is None:
-            metric_name = f"R² {self._SCORE_OR_LOSS_ICONS['r2']}"
-
         return self._compute_metric_scores(
             metrics.r2_score,
             X=X,
@@ -1094,7 +1079,6 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
             data_source=data_source,
             data_source_hash=data_source_hash,
             response_method="predict",
-            metric_name=metric_name,
             multioutput=multioutput,
         )
 
@@ -1131,7 +1115,7 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
 
         Returns
         -------
-        pd.DataFrame
+        float or ndarray of shape (n_outputs,)
             The root mean squared error.
 
         Examples
@@ -1152,8 +1136,7 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
         ...     y_test=y_test,
         ... )
         >>> report.metrics.rmse()
-        Metric  RMSE (↘︎)
-        Ridge   56.5...
+        np.float64(56.5...)
         """
         return self._rmse(
             data_source=data_source,
@@ -1170,7 +1153,6 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
         data_source_hash=None,
         X=None,
         y=None,
-        metric_name=None,
         multioutput="raw_values",
     ):
         """Private interface of `rmse` to be able to pass `data_source_hash`.
@@ -1178,12 +1160,7 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
         `data_source_hash` is either an `int` when we already computed the hash
         and are able to pass it around or `None` and thus trigger its computation
         in the underlying process.
-
-        `metric_name` allows to overwrite the default name of the metric in the report.
         """
-        if metric_name is None:
-            metric_name = f"RMSE {self._SCORE_OR_LOSS_ICONS['rmse']}"
-
         return self._compute_metric_scores(
             metrics.root_mean_squared_error,
             X=X,
@@ -1191,7 +1168,6 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
             data_source=data_source,
             data_source_hash=data_source_hash,
             response_method="predict",
-            metric_name=metric_name,
             multioutput=multioutput,
         )
 
@@ -1200,7 +1176,6 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
         metric_function,
         response_method,
         *,
-        metric_name=None,
         data_source="test",
         X=None,
         y=None,
@@ -1228,10 +1203,6 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
             values are: `predict`, `predict_proba`, `predict_log_proba`, and
             `decision_function`.
 
-        metric_name : str, default=None
-            The name of the metric. If not provided, it will be inferred from the
-            metric function.
-
         data_source : {"test", "train", "X_y"}, default="test"
             The data source to use.
 
@@ -1252,8 +1223,8 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
 
         Returns
         -------
-        pd.DataFrame
-            The custom metric.
+        float, dict, or ndarray of shape (n_outputs,)
+            The custom metric. The output type depends on the metric function.
 
         Examples
         --------
@@ -1276,10 +1247,15 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
         >>> report.metrics.custom_metric(
         ...     metric_function=mean_absolute_error,
         ...     response_method="predict",
-        ...     metric_name="MAE (↗︎)",
         ... )
-        Metric        MAE (↗︎)
-        Ridge   44.9...
+        44.9...
+        >>> def metric_function(y_true, y_pred):
+        ...     return {"output": float(mean_absolute_error(y_true, y_pred))}
+        >>> report.metrics.custom_metric(
+        ...     metric_function=metric_function,
+        ...     response_method="predict",
+        ... )
+        {'output': 44.9...}
         """
         return self._custom_metric(
             data_source=data_source,
@@ -1288,7 +1264,6 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
             y=y,
             metric_function=metric_function,
             response_method=response_method,
-            metric_name=metric_name,
             **kwargs,
         )
 
@@ -1301,7 +1276,6 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
         y=None,
         metric_function=None,
         response_method=None,
-        metric_name=None,
         **kwargs,
     ):
         """Private interface of `custom_metric` to be able to pass `data_source_hash`.
@@ -1317,7 +1291,6 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
             data_source=data_source,
             data_source_hash=data_source_hash,
             response_method=response_method,
-            metric_name=metric_name,
             **kwargs,
         )
 
@@ -1347,15 +1320,16 @@ class _MetricsAccessor(_BaseAccessor, DirNamesMixin):
         """Override format method for metrics-specific naming."""
         method_name = f"{name}(...)"
         method_name = method_name.ljust(22)
-        if name in self._SCORE_OR_LOSS_ICONS and self._SCORE_OR_LOSS_ICONS[name] in (
-            "(↗︎)",
-            "(↘︎)",
-        ):
-            if self._SCORE_OR_LOSS_ICONS[name] == "(↗︎)":
-                method_name += f"[cyan]{self._SCORE_OR_LOSS_ICONS[name]}[/cyan]"
+        if name in self._SCORE_OR_LOSS_INFO and self._SCORE_OR_LOSS_INFO[name][
+            "icon"
+        ] in ("(↗︎)", "(↘︎)"):
+            if self._SCORE_OR_LOSS_INFO[name]["icon"] == "(↗︎)":
+                method_name += f"[cyan]{self._SCORE_OR_LOSS_INFO[name]['icon']}[/cyan]"
                 return method_name.ljust(43)
             else:  # (↘︎)
-                method_name += f"[orange1]{self._SCORE_OR_LOSS_ICONS[name]}[/orange1]"
+                method_name += (
+                    f"[orange1]{self._SCORE_OR_LOSS_INFO[name]['icon']}[/orange1]"
+                )
                 return method_name.ljust(49)
         else:
             return method_name.ljust(29)
diff --git a/skore/src/skore/sklearn/_estimator/metrics_accessor.pyi b/skore/src/skore/sklearn/_estimator/metrics_accessor.pyi
index 3dfb8a1..296a1d4 100644
--- a/skore/src/skore/sklearn/_estimator/metrics_accessor.pyi
+++ b/skore/src/skore/sklearn/_estimator/metrics_accessor.pyi
@@ -55,7 +55,7 @@ class _PlotMetricsAccessor(_BaseAccessor):
     ) -> PredictionErrorDisplay: ...
 
 class _MetricsAccessor(_BaseAccessor):
-    _SCORE_OR_LOSS_ICONS: dict[str, str]
+    _SCORE_OR_LOSS_INFO: dict[str, dict[str, str]]
     plot: _PlotMetricsAccessor
 
     def _compute_metric_scores(
diff --git a/skore/src/skore/sklearn/find_ml_task.py b/skore/src/skore/sklearn/find_ml_task.py
index 9941964..d311219 100644
--- a/skore/src/skore/sklearn/find_ml_task.py
+++ b/skore/src/skore/sklearn/find_ml_task.py
@@ -36,6 +36,7 @@ def _find_ml_task(y, estimator=None) -> MLTask:
     Examples
     --------
     >>> import numpy
+    >>> from skore.sklearn.find_ml_task import _find_ml_task
 
     # Discrete values, not sequential
     >>> _find_ml_task(numpy.array([1, 5, 9]))


Test patch:
diff --git a/skore/tests/unit/sklearn/test_cross_validation.py b/skore/tests/unit/sklearn/test_cross_validation.py
index 0fb95e8..f16dba3 100644
--- a/skore/tests/unit/sklearn/test_cross_validation.py
+++ b/skore/tests/unit/sklearn/test_cross_validation.py
@@ -4,7 +4,7 @@ import joblib
 import numpy as np
 import pandas as pd
 import pytest
-from sklearn.base import BaseEstimator, clone
+from sklearn.base import BaseEstimator, ClassifierMixin, clone
 from sklearn.datasets import make_classification, make_regression
 from sklearn.ensemble import RandomForestClassifier
 from sklearn.exceptions import NotFittedError
@@ -556,7 +556,7 @@ def test_cross_validation_report_report_metrics_scoring_kwargs(
     result = report.metrics.report_metrics(scoring_kwargs={"average": None})
     assert result.shape == (2, 10)
     assert isinstance(result.columns, pd.MultiIndex)
-    assert result.columns.names == ["Metric", "Class label"]
+    assert result.columns.names == ["Metric", "Label / Average"]
 
 
 @pytest.mark.parametrize(
@@ -732,7 +732,7 @@ def test_cross_validation_report_interrupted(
     """Check that we can interrupt cross-validation without losing all
     data."""
 
-    class MockEstimator(BaseEstimator):
+    class MockEstimator(ClassifierMixin, BaseEstimator):
         def __init__(self, n_call=0, fail_after_n_clone=3):
             self.n_call = n_call
             self.fail_after_n_clone = fail_after_n_clone
@@ -740,6 +740,7 @@ def test_cross_validation_report_interrupted(
         def fit(self, X, y):
             if self.n_call > self.fail_after_n_clone:
                 raise error
+            self.classes_ = np.unique(y)
             return self
 
         def __sklearn_clone__(self):
diff --git a/skore/tests/unit/sklearn/test_estimator.py b/skore/tests/unit/sklearn/test_estimator.py
index 7740f6e..0a1370a 100644
--- a/skore/tests/unit/sklearn/test_estimator.py
+++ b/skore/tests/unit/sklearn/test_estimator.py
@@ -1,5 +1,6 @@
 import re
 from copy import deepcopy
+from numbers import Real
 
 import joblib
 import numpy as np
@@ -487,9 +488,7 @@ def test_estimator_report_metrics_repr(binary_classification_data):
     assert "report.metrics.help()" in repr_str
 
 
-@pytest.mark.parametrize(
-    "metric", ["accuracy", "precision", "recall", "brier_score", "roc_auc", "log_loss"]
-)
+@pytest.mark.parametrize("metric", ["accuracy", "brier_score", "roc_auc", "log_loss"])
 def test_estimator_report_metrics_binary_classification(
     binary_classification_data, metric
 ):
@@ -500,10 +499,10 @@ def test_estimator_report_metrics_binary_classification(
     report = EstimatorReport(estimator, X_test=X_test, y_test=y_test)
     assert hasattr(report.metrics, metric)
     result = getattr(report.metrics, metric)()
-    assert isinstance(result, pd.DataFrame)
+    assert isinstance(result, float)
     # check that we hit the cache
     result_with_cache = getattr(report.metrics, metric)()
-    pd.testing.assert_frame_equal(result, result_with_cache)
+    assert result == pytest.approx(result_with_cache)
 
     # check that something was written to the cache
     assert report._cache != {}
@@ -514,8 +513,38 @@ def test_estimator_report_metrics_binary_classification(
     result_external_data = getattr(report.metrics, metric)(
         data_source="X_y", X=X_test, y=y_test
     )
-    assert isinstance(result_external_data, pd.DataFrame)
-    pd.testing.assert_frame_equal(result, result_external_data)
+    assert isinstance(result_external_data, float)
+    assert result == pytest.approx(result_external_data)
+    assert report._cache != {}
+
+
+@pytest.mark.parametrize("metric", ["precision", "recall"])
+def test_estimator_report_metrics_binary_classification_pr(
+    binary_classification_data, metric
+):
+    """Check the behaviour of the precision and recall metrics available for binary
+    classification.
+    """
+    estimator, X_test, y_test = binary_classification_data
+    report = EstimatorReport(estimator, X_test=X_test, y_test=y_test)
+    assert hasattr(report.metrics, metric)
+    result = getattr(report.metrics, metric)()
+    assert isinstance(result, dict)
+    # check that we hit the cache
+    result_with_cache = getattr(report.metrics, metric)()
+    assert result == result_with_cache
+
+    # check that something was written to the cache
+    assert report._cache != {}
+    report.clear_cache()
+
+    # check that passing using data outside from the report works and that we they
+    # don't come from the cache
+    result_external_data = getattr(report.metrics, metric)(
+        data_source="X_y", X=X_test, y=y_test
+    )
+    assert isinstance(result_external_data, dict)
+    assert result == result_external_data
     assert report._cache != {}
 
 
@@ -526,10 +555,10 @@ def test_estimator_report_metrics_regression(regression_data, metric):
     report = EstimatorReport(estimator, X_test=X_test, y_test=y_test)
     assert hasattr(report.metrics, metric)
     result = getattr(report.metrics, metric)()
-    assert isinstance(result, pd.DataFrame)
+    assert isinstance(result, float)
     # check that we hit the cache
     result_with_cache = getattr(report.metrics, metric)()
-    pd.testing.assert_frame_equal(result, result_with_cache)
+    assert result == pytest.approx(result_with_cache)
 
     # check that something was written to the cache
     assert report._cache != {}
@@ -540,8 +569,8 @@ def test_estimator_report_metrics_regression(regression_data, metric):
     result_external_data = getattr(report.metrics, metric)(
         data_source="X_y", X=X_test, y=y_test
     )
-    assert isinstance(result_external_data, pd.DataFrame)
-    pd.testing.assert_frame_equal(result, result_external_data)
+    assert isinstance(result_external_data, float)
+    assert result == pytest.approx(result_external_data)
     assert report._cache != {}
 
 
@@ -684,7 +713,7 @@ def test_estimator_report_report_metrics_scoring_kwargs(
     result = report.metrics.report_metrics(scoring_kwargs={"average": None})
     assert result.shape == (1, 10)
     assert isinstance(result.columns, pd.MultiIndex)
-    assert result.columns.names == ["Metric", "Class label"]
+    assert result.columns.names == ["Metric", "Label / Average"]
 
 
 @pytest.mark.parametrize(
@@ -745,7 +774,7 @@ def test_estimator_report_interaction_cache_metrics(regression_multioutput_data)
     assert (
         not should_raise
     ), f"The value {multioutput} should be stored in one of the cache keys"
-    assert result_r2_raw_values.shape == (1, 2)
+    assert result_r2_raw_values.shape == (2,)
 
     multioutput = "uniform_average"
     result_r2_uniform_average = report.metrics.r2(multioutput=multioutput)
@@ -757,7 +786,7 @@ def test_estimator_report_interaction_cache_metrics(regression_multioutput_data)
     assert (
         not should_raise
     ), f"The value {multioutput} should be stored in one of the cache keys"
-    assert result_r2_uniform_average.shape == (1, 1)
+    assert isinstance(result_r2_uniform_average, float)
 
 
 def test_estimator_report_custom_metric(regression_data):
@@ -772,7 +801,6 @@ def test_estimator_report_custom_metric(regression_data):
     threshold = 1
     result = report.metrics.custom_metric(
         metric_function=custom_metric,
-        metric_name="Custom Metric",
         response_method="predict",
         threshold=threshold,
     )
@@ -785,15 +813,14 @@ def test_estimator_report_custom_metric(regression_data):
         not should_raise
     ), f"The value {threshold} should be stored in one of the cache keys"
 
-    assert result.columns.tolist() == ["Custom Metric"]
-    assert result.to_numpy()[0, 0] == pytest.approx(
+    assert isinstance(result, float)
+    assert result == pytest.approx(
         custom_metric(y_test, estimator.predict(X_test), threshold)
     )
 
     threshold = 100
     result = report.metrics.custom_metric(
         metric_function=custom_metric,
-        metric_name="Custom Metric",
         response_method="predict",
         threshold=threshold,
     )
@@ -806,8 +833,8 @@ def test_estimator_report_custom_metric(regression_data):
         not should_raise
     ), f"The value {threshold} should be stored in one of the cache keys"
 
-    assert result.columns.tolist() == ["Custom Metric"]
-    assert result.to_numpy()[0, 0] == pytest.approx(
+    assert isinstance(result, float)
+    assert result == pytest.approx(
         custom_metric(y_test, estimator.predict(X_test), threshold)
     )
 
@@ -838,7 +865,6 @@ def test_estimator_report_custom_function_kwargs_numpy_array(regression_data):
 
     result = report.metrics.custom_metric(
         metric_function=custom_metric,
-        metric_name="Custom Metric",
         response_method="predict",
         some_weights=weights,
     )
@@ -851,8 +877,8 @@ def test_estimator_report_custom_function_kwargs_numpy_array(regression_data):
         not should_raise
     ), "The hash of the weights should be stored in one of the cache keys"
 
-    assert result.columns.tolist() == ["Custom Metric"]
-    assert result.to_numpy()[0, 0] == pytest.approx(
+    assert isinstance(result, float)
+    assert result == pytest.approx(
         custom_metric(y_test, estimator.predict(X_test), weights)
     )
 
@@ -940,11 +966,10 @@ def test_estimator_report_custom_metric_compatible_estimator(
     report = EstimatorReport(estimator, fit=False, X_test=X_test, y_test=y_test)
     result = report.metrics.custom_metric(
         metric_function=lambda y_true, y_pred: 1,
-        metric_name="Custom Metric",
         response_method="predict",
     )
-    assert result.columns.tolist() == ["Custom Metric"]
-    assert result.to_numpy()[0, 0] == 1
+    assert isinstance(result, Real)
+    assert result == pytest.approx(1)
 
 
 @pytest.mark.parametrize(