Repository: aeon-toolkit/aeon
Pull request #1141 (2024-02-07 07:55:57)

[ENH, MNT] Changes to docs in significance plots and stats.
This PR solves some comments left by @MatthewMiddlehurst  in #954.

aeon-actions-bot[bot]: 
## Thank you for contributing to `aeon`


I would have added the following labels to this PR based on the changes made: **[ $\\color{#FBCA04}{\\textsf{visualisation}}$ ]**, however some package labels are already present.

The [Checks](https://github.com/aeon-toolkit/aeon/pull/1141/checks) tab will show the status of our automated tests. You can click on individual test runs in the tab or "Details" in the panel below to see more information if there is a failure.

If our `pre-commit` code quality check fails, any trivial fixes will automatically be pushed to your PR unless it is a draft.

Don't hesitate to ask questions on the `aeon` [Slack](
https://join.slack.com/t/aeon-toolkit/shared_invite/zt-22vwvut29-HDpCu~7VBUozyfL_8j3dLA) channel if you have any.
    

Patch:
diff --git a/aeon/benchmarking/forecasting.py b/aeon/benchmarking/forecasting.py
index 451d66c6..283271a5 100644
--- a/aeon/benchmarking/forecasting.py
+++ b/aeon/benchmarking/forecasting.py
@@ -7,13 +7,12 @@ from aeon.benchmarking.benchmarks import BaseBenchmark
 from aeon.forecasting.base import BaseForecaster
 from aeon.forecasting.model_evaluation import evaluate
 from aeon.forecasting.model_selection._split import BaseSplitter
-from aeon.performance_metrics.base import BaseMetric
 
 
 def forecasting_validation(
     dataset_loader: Callable,
     cv_splitter: BaseSplitter,
-    scorers: List[BaseMetric],
+    scorers: List[Callable],
     estimator: BaseForecaster,
     **kwargs,
 ) -> Dict[str, Union[float, str]]:
@@ -25,8 +24,8 @@ def forecasting_validation(
         A function which returns a dataset, like from `aeon.datasets`.
     cv_splitter : BaseSplitter object
         Splitter used for generating validation folds.
-    scorers : a list of BaseMetric objects
-        Each BaseMetric output will be included in the results.
+    scorers : a list of Callable scoring functions
+        Each scoring metric output will be included in the results.
     estimator : BaseForecaster object
         Estimator to benchmark.
 
@@ -37,8 +36,7 @@ def forecasting_validation(
     y = dataset_loader()
     results = {}
     for scorer in scorers:
-        scorer_name = scorer.name
-        # TODO re-write evaluate to allow multiple scorers, to avoid recomputation
+        scorer_name = scorer.__name__
         scores_df = evaluate(forecaster=estimator, y=y, cv=cv_splitter, scoring=scorer)
         for ix, row in scores_df.iterrows():
             results[f"{scorer_name}_fold_{ix}_test"] = row[f"test_{scorer_name}"]
@@ -50,7 +48,7 @@ def forecasting_validation(
 def _factory_forecasting_validation(
     dataset_loader: Callable,
     cv_splitter: BaseSplitter,
-    scorers: List[BaseMetric],
+    scorers: List[Callable],
 ) -> Callable:
     """Build validation func which just takes a forecasting estimator."""
     return functools.partial(
@@ -73,7 +71,7 @@ class ForecastingBenchmark(BaseBenchmark):
         self,
         dataset_loader: Callable,
         cv_splitter: BaseSplitter,
-        scorers: List[BaseMetric],
+        scorers: List[Callable],
         task_id: Optional[str] = None,
     ):
         """Register a forecasting task to the benchmark.
@@ -84,8 +82,8 @@ class ForecastingBenchmark(BaseBenchmark):
             A function which returns a dataset, like from `aeon.datasets`.
         cv_splitter : BaseSplitter object
             Splitter used for generating validation folds.
-        scorers : a list of BaseMetric objects
-            Each BaseMetric output will be included in the results.
+        scorers : a list of Callable scoring functions
+            Each scoring function output will be included in the results.
         task_id : str, optional (default=None)
             Identifier for the benchmark task. If none given then uses dataset loader
             name combined with cv_splitter class name.
diff --git a/aeon/forecasting/model_evaluation/_functions.py b/aeon/forecasting/model_evaluation/_functions.py
index 97ee024b..688b65a9 100644
--- a/aeon/forecasting/model_evaluation/_functions.py
+++ b/aeon/forecasting/model_evaluation/_functions.py
@@ -149,15 +149,16 @@ def _evaluate_window(
         # predict
         start_pred = time.perf_counter()
 
-        if hasattr(scoring, "metric_args"):
-            metric_args = scoring.metric_args
+        scitype = None
+        metric_args = {}
+        from aeon.performance_metrics.forecasting.probabilistic import (
+            _BaseProbaForecastingErrorMetric,
+        )
 
-        try:
+        if isinstance(scoring, _BaseProbaForecastingErrorMetric):
+            if hasattr(scoring, "metric_args"):
+                metric_args = scoring.metric_args
             scitype = scoring.get_tag("y_input_type_pred")
-        except ValueError:
-            # If no scitype exists then metric is not proba and no args needed
-            scitype = None
-            metric_args = {}
 
         y_pred = eval(pred_type[scitype])(fh, X_test, **metric_args)
         pred_time = time.perf_counter() - start_pred
@@ -236,9 +237,9 @@ def evaluate(
         "refit" = forecaster is refitted to each training window
         "update" = forecaster is updated with training window data, in sequence provided
         "no-update_params" = fit to first training window, re-used without fit or update
-    scoring : subclass of aeon.performance_metrics.BaseMetric or list of same,
-        default=None. Used to get a score function that takes y_pred and y_test
-        arguments and accept y_train as keyword argument.
+    scoring : Callable or None, default=None
+        Function in aeon.performance_metrics. Used to get a score function that takes
+        y_pred and y_test arguments and accept y_train as keyword argument.
         If None, then uses scoring = MeanAbsolutePercentageError().
     return_data : bool, default=False
         Returns three additional columns in the DataFrame, by default False.
@@ -303,17 +304,17 @@ def evaluate(
         i.e., point forecast metrics, interval metrics, quantile foreast metrics.
         https://www.aeon-toolkit.org/en/stable/api_reference/performance_metrics.html?highlight=metrics
         To evaluate estimators using a specific metric, provide them to the scoring arg.
-    >>> from aeon.performance_metrics.forecasting import MeanAbsoluteError
-    >>> loss = MeanAbsoluteError()
+    >>> from aeon.performance_metrics.forecasting import mean_absolute_error as loss
     >>> results = evaluate(forecaster=forecaster, y=y, cv=cv, scoring=loss)
 
         Optionally, users can provide a list of metrics to `scoring` argument.
-    >>> from aeon.performance_metrics.forecasting import MeanSquaredError
+    >>> from aeon.performance_metrics.forecasting import mean_absolute_error as loss
+    >>> from aeon.performance_metrics.forecasting import mean_squared_error as loss2
     >>> results = evaluate(
     ...     forecaster=forecaster,
     ...     y=y,
     ...     cv=cv,
-    ...     scoring=[MeanSquaredError(square_root=True), MeanAbsoluteError()],
+    ...     scoring=[loss, loss2],
     ... )
 
         An example of an interval metric is the `PinballLoss`.
@@ -366,11 +367,10 @@ def evaluate(
                 f"Expected X dtype {ALLOWED_SCITYPES!r}. Got {type(X)} instead."
             )
         X = convert_to(X, to_type=PANDAS_MTYPES)
-
     score_name = (
-        f"test_{scoring.name}"
+        f"test_{scoring.__name__}"
         if not isinstance(scoring, List)
-        else f"test_{scoring[0].name}"
+        else f"test_{scoring[0].__name__}"
     )
     cutoff_dtype = str(y.index.dtype)
     _evaluate_window_kwargs = {
@@ -464,9 +464,9 @@ def evaluate(
 
     if isinstance(scoring, List):
         for s in scoring[1:]:
-            results[f"test_{s.name}"] = np.nan
+            results[f"test_{s.__name__}"] = np.nan
             for row in range(len(results)):
-                results[f"test_{s.name}"].iloc[row] = s(
+                results[f"test_{s.__name__}"].iloc[row] = s(
                     results["y_test"].iloc[row],
                     results["y_pred"].iloc[row],
                     y_train=results["y_train"].iloc[row],
diff --git a/aeon/forecasting/model_selection/_tune.py b/aeon/forecasting/model_selection/_tune.py
index a6d57ee0..997a6260 100644
--- a/aeon/forecasting/model_selection/_tune.py
+++ b/aeon/forecasting/model_selection/_tune.py
@@ -141,7 +141,7 @@ class BaseGridSearch(_DelegatedForecaster):
         cv = check_cv(self.cv)
 
         scoring = check_scoring(self.scoring)
-        scoring_name = f"test_{scoring.name}"
+        scoring_name = f"test_{scoring.__name__}"
 
         def _fit_and_score(params):
             # Clone forecaster.
@@ -208,7 +208,7 @@ class BaseGridSearch(_DelegatedForecaster):
 
         # Rank results, according to whether greater is better for the given scoring.
         results[f"rank_{scoring_name}"] = results.loc[:, f"mean_{scoring_name}"].rank(
-            ascending=scoring.get_tag("lower_is_better")
+            ascending=True
         )
 
         self.cv_results_ = results
@@ -231,9 +231,7 @@ class BaseGridSearch(_DelegatedForecaster):
             self.best_forecaster_.fit(y, X, fh)
 
         # Sort values according to rank
-        results = results.sort_values(
-            by=f"rank_{scoring_name}", ascending=scoring.get_tag("lower_is_better")
-        )
+        results = results.sort_values(by=f"rank_{scoring_name}", ascending=True)
         # Select n best forecaster
         self.n_best_forecasters_ = []
         self.n_best_scores_ = []
@@ -526,19 +524,19 @@ class ForecastingGridSearchCV(BaseGridSearch):
         from aeon.forecasting.model_selection._split import SingleWindowSplitter
         from aeon.forecasting.naive import NaiveForecaster
         from aeon.forecasting.trend import PolynomialTrendForecaster
-        from aeon.performance_metrics.forecasting import MeanAbsolutePercentageError
+        from aeon.performance_metrics.forecasting import mean_absolute_error
 
         params = {
             "forecaster": NaiveForecaster(strategy="mean"),
             "cv": SingleWindowSplitter(fh=1),
             "param_grid": {"window_length": [2, 5]},
-            "scoring": MeanAbsolutePercentageError(symmetric=True),
+            "scoring": mean_absolute_error,
         }
         params2 = {
             "forecaster": PolynomialTrendForecaster(),
             "cv": SingleWindowSplitter(fh=1),
             "param_grid": {"degree": [1, 2]},
-            "scoring": MeanAbsolutePercentageError(symmetric=True),
+            "scoring": mean_absolute_error,
             "update_behaviour": "inner_only",
         }
         return [params, params2]
@@ -702,20 +700,17 @@ class ForecastingRandomizedSearchCV(BaseGridSearch):
         from aeon.forecasting.model_selection._split import SingleWindowSplitter
         from aeon.forecasting.naive import NaiveForecaster
         from aeon.forecasting.trend import PolynomialTrendForecaster
-        from aeon.performance_metrics.forecasting import MeanAbsolutePercentageError
 
         params = {
             "forecaster": NaiveForecaster(strategy="mean"),
             "cv": SingleWindowSplitter(fh=1),
             "param_distributions": {"window_length": [2, 5]},
-            "scoring": MeanAbsolutePercentageError(symmetric=True),
         }
 
         params2 = {
             "forecaster": PolynomialTrendForecaster(),
             "cv": SingleWindowSplitter(fh=1),
             "param_distributions": {"degree": [1, 2]},
-            "scoring": MeanAbsolutePercentageError(symmetric=True),
             "update_behaviour": "inner_only",
         }
 
diff --git a/aeon/performance_metrics/forecasting/_classes.py b/aeon/performance_metrics/forecasting/_classes.py
index 704bebe4..6b826a25 100644
--- a/aeon/performance_metrics/forecasting/_classes.py
+++ b/aeon/performance_metrics/forecasting/_classes.py
@@ -11,6 +11,7 @@ from warnings import warn
 
 import numpy as np
 import pandas as pd
+from deprecated.sphinx import deprecated
 from sklearn.utils import check_array
 
 from aeon.datatypes import VectorizedDF, check_is_scitype, convert_to
@@ -66,23 +67,14 @@ __all__ = [
 ]
 
 
-def _coerce_to_scalar(obj):
-    """Coerce obj to scalar, from polymorphic input scalar or pandas."""
-    if isinstance(obj, pd.DataFrame):
-        assert len(obj) == 1
-        assert len(obj.columns) == 1
-        return obj.iloc[0, 0]
-    if isinstance(obj, pd.Series):
-        assert len(obj) == 1
-        return obj.iloc[0]
-    return obj
-
-
-def _coerce_to_df(obj):
-    """Coerce to pd.DataFrame, from polymorphic input scalar or pandas."""
-    return pd.DataFrame(obj)
-
-
+# TODO: remove in v0.8.0
+@deprecated(
+    version="0.6.0",
+    reason="BaseForecastingErrorMetric and all subclasses will be removed from "
+    "performance_metric package in v0.8.0. Please use the functions rather "
+    "than the functor classes.",
+    category=FutureWarning,
+)
 class BaseForecastingErrorMetric(BaseMetric):
     """Base class for defining forecasting error metrics in aeon.
 
@@ -123,7 +115,7 @@ class BaseForecastingErrorMetric(BaseMetric):
 
         if not hasattr(self, "name"):
             self.name = type(self).__name__
-
+        self.__name__ = self.name
         super().__init__()
 
     def __call__(self, y_true, y_pred, **kwargs):
@@ -216,9 +208,15 @@ class BaseForecastingErrorMetric(BaseMetric):
             and isinstance(multioutput, str)
             and multioutput == "uniform_average"
         ):
-            out_df = _coerce_to_scalar(out_df)
+            if isinstance(out_df, pd.DataFrame):
+                assert len(out_df) == 1
+                assert len(out_df.columns) == 1
+                out_df = out_df.iloc[0, 0]
+            if isinstance(out_df, pd.Series):
+                assert len(out_df) == 1
+                out_df = out_df.iloc[0]
         if multilevel == "raw_values":
-            out_df = _coerce_to_df(out_df)
+            out_df = pd.DataFrame(out_df)
 
         return out_df
 
diff --git a/aeon/performance_metrics/forecasting/probabilistic/_classes.py b/aeon/performance_metrics/forecasting/probabilistic/_classes.py
index fc063950..93b7a432 100644
--- a/aeon/performance_metrics/forecasting/probabilistic/_classes.py
+++ b/aeon/performance_metrics/forecasting/probabilistic/_classes.py
@@ -1,12 +1,437 @@
+from copy import deepcopy
 from logging import warning
+from warnings import warn
 
 import numpy as np
 import pandas as pd
 from pandas.api.types import is_numeric_dtype
 from sklearn.utils import check_array, check_consistent_length
 
-from aeon.datatypes import check_is_scitype, convert
-from aeon.performance_metrics.forecasting._classes import BaseForecastingErrorMetric
+from aeon.datatypes import VectorizedDF, check_is_scitype, convert, convert_to
+from aeon.performance_metrics.base import BaseMetric
+
+
+class BaseForecastingErrorMetric(BaseMetric):
+    """Base class for defining forecasting error metrics in aeon.
+
+    Extends aeon's BaseMetric to the forecasting interface. Forecasting error
+    metrics measure the error (loss) between forecasts and true values.
+
+    `multioutput` and `multilevel` parameters can be used to control averaging
+    across variables (`multioutput`) and (non-temporal) hierarchy levels (`multilevel`).
+
+    Parameters
+    ----------
+    multioutput : {'raw_values', 'uniform_average'}  or array-like of shape \
+            (n_outputs,), default='uniform_average'
+        Defines whether and how to aggregate metric for across variables.
+        If 'uniform_average' (default), errors are mean-averaged across variables.
+        If array-like, errors are weighted averaged across variables, values as weights.
+        If 'raw_values', does not average errors across variables, columns are retained.
+
+    multilevel : {'raw_values', 'uniform_average', 'uniform_average_time'}
+        Defines how to aggregate metric for hierarchical data (with levels).
+        If 'uniform_average' (default), errors are mean-averaged across levels.
+        If 'uniform_average_time', errors are mean-averaged across rows.
+        If 'raw_values', does not average errors across levels, hierarchy is retained.
+    """
+
+    _tags = {
+        "requires-y-train": False,
+        "requires-y-pred-benchmark": False,
+        "univariate-only": False,
+        "lower_is_better": True,
+        # "y_inner_type": ["pd.DataFrame", "pd-multiindex", "pd_multiindex_hier"]
+        "inner_implements_multilevel": False,
+    }
+
+    def __init__(self, multioutput="uniform_average", multilevel="uniform_average"):
+        self.multioutput = multioutput
+        self.multilevel = multilevel
+
+        if not hasattr(self, "name"):
+            self.name = type(self).__name__
+        self.__name__ = self.name
+        super().__init__()
+
+    def __call__(self, y_true, y_pred, **kwargs):
+        """Calculate metric value using underlying metric function.
+
+        Parameters
+        ----------
+        y_true : time series in aeon compatible data container format
+            Ground truth (correct) target values
+            y can be in one of the following formats:
+            Series type: pd.Series, pd.DataFrame, or np.ndarray (1D or 2D)
+            Panel type: pd.DataFrame with 2-level row MultiIndex,
+                3D np.ndarray, list of Series pd.DataFrame, or nested pd.DataFrame
+            Hierarchical type: pd.DataFrame with 3 or more level row MultiIndex
+        y_pred :time series in aeon compatible data container format
+            Forecasted values to evaluate
+            must be of same format as y_true, same indices and columns if indexed
+
+        Returns
+        -------
+        loss : float, np.ndarray, or pd.DataFrame
+            Calculated metric, averaged or by variable.
+            float if self.multioutput="uniform_average" or array-like
+                and self.multilevel="uniform_average" or "uniform_average_time"
+                value is metric averaged over variables and levels (see class docstring)
+            np.ndarray of shape (y_true.columns,) if self.multioutput="raw_values"
+                and self.multilevel="uniform_average" or "uniform_average_time"
+                i-th entry is metric calculated for i-th variable
+            pd.DataFrame if self.multilevel=raw.values
+                of shape (n_levels, ) if self.multioutput = "uniform_average" or array
+                of shape (n_levels, y_true.columns) if self.multioutput="raw_values"
+                metric is applied per level, row averaging (yes/no) as in multioutput
+        """
+        return self.evaluate(y_true, y_pred, **kwargs)
+
+    def evaluate(self, y_true, y_pred, **kwargs):
+        """Evaluate the desired metric on given inputs.
+
+        Parameters
+        ----------
+        y_true : time series in aeon compatible data container format
+            Ground truth (correct) target values
+            y can be in one of the following formats:
+            Series type: pd.Series, pd.DataFrame, or np.ndarray (1D or 2D)
+            Panel type: pd.DataFrame with 2-level row MultiIndex,
+                3D np.ndarray, list of Series pd.DataFrame, or nested pd.DataFrame
+            Hierarchical type: pd.DataFrame with 3 or more level row MultiIndex
+        y_pred :time series in aeon compatible data container format
+            Forecasted values to evaluate
+            must be of same format as y_true, same indices and columns if indexed
+
+        Returns
+        -------
+        loss : float, np.ndarray, or pd.DataFrame
+            Calculated metric, averaged or by variable.
+            float if self.multioutput="uniform_average" or array-like
+                and self.multilevel="uniform_average" or "uniform_average_time"
+                value is metric averaged over variables and levels (see class docstring)
+            np.ndarray of shape (y_true.columns,) if self.multioutput="raw_values"
+                and self.multilevel="uniform_average" or "uniform_average_time"
+                i-th entry is metric calculated for i-th variable
+            pd.DataFrame if self.multilevel=raw.values
+                of shape (n_levels, ) if self.multioutput = "uniform_average" or array
+                of shape (n_levels, y_true.columns) if self.multioutput="raw_values"
+                metric is applied per level, row averaging (yes/no) as in multioutput
+        """
+        multioutput = self.multioutput
+        multilevel = self.multilevel
+        # Input checks and conversions
+        y_true_inner, y_pred_inner, multioutput, multilevel, kwargs = self._check_ys(
+            y_true, y_pred, multioutput, multilevel, **kwargs
+        )
+
+        requires_vectorization = isinstance(y_true_inner, VectorizedDF)
+        if not requires_vectorization:
+            # pass to inner function
+            out_df = self._evaluate(y_true=y_true_inner, y_pred=y_pred_inner, **kwargs)
+        else:
+            out_df = self._evaluate_vectorized(
+                y_true=y_true_inner, y_pred=y_pred_inner, **kwargs
+            )
+            if multilevel == "uniform_average":
+                out_df = out_df.mean(axis=0)
+                # if level is averaged, but not variables, return numpy
+                if isinstance(multioutput, str) and multioutput == "raw_values":
+                    out_df = out_df.values
+
+        if (
+            multilevel == "uniform_average"
+            and isinstance(multioutput, str)
+            and multioutput == "uniform_average"
+        ):
+            if isinstance(out_df, pd.DataFrame):
+                assert len(out_df) == 1
+                assert len(out_df.columns) == 1
+                out_df = out_df.iloc[0, 0]
+            if isinstance(out_df, pd.Series):
+                assert len(out_df) == 1
+                out_df = out_df.iloc[0]
+        if multilevel == "raw_values":
+            out_df = pd.DataFrame(out_df)
+
+        return out_df
+
+    def _evaluate(self, y_true, y_pred, **kwargs):
+        """Evaluate the desired metric on given inputs.
+
+        private _evaluate containing core logic, called from evaluate
+
+        By default this uses evaluate_by_index, taking arithmetic mean over time points.
+
+        Parameters
+        ----------
+        y_true : time series in aeon compatible data container format
+            Ground truth (correct) target values
+            y can be in one of the following formats:
+            Series type: pd.Series, pd.DataFrame, or np.ndarray (1D or 2D)
+            Panel type: pd.DataFrame with 2-level row MultiIndex,
+                3D np.ndarray, list of Series pd.DataFrame, or nested pd.DataFrame
+            Hierarchical type: pd.DataFrame with 3 or more level row MultiIndex
+        y_pred :time series in aeon compatible data container format
+            Forecasted values to evaluate
+            must be of same format as y_true, same indices and columns if indexed
+
+        Returns
+        -------
+        loss : float or np.ndarray
+            Calculated metric, averaged or by variable.
+            float if self.multioutput="uniform_average" or array-like
+                value is metric averaged over variables (see class docstring)
+            np.ndarray of shape (y_true.columns,) if self.multioutput="raw_values"
+                i-th entry is metric calculated for i-th variable
+        """
+        # multioutput = self.multioutput
+        # multilevel = self.multilevel
+        try:
+            index_df = self._evaluate_by_index(y_true, y_pred, **kwargs)
+            return index_df.mean(axis=0)
+        except RecursionError:
+            RecursionError("Must implement one of _evaluate or _evaluate_by_index")
+
+    def _evaluate_vectorized(self, y_true, y_pred, **kwargs):
+        """Vectorized version of _evaluate.
+
+        Runs _evaluate for all instances in y_true, y_pred,
+        and returns results in a hierarchical pandas.DataFrame.
+
+        Parameters
+        ----------
+        y_true : pandas.DataFrame with MultiIndex, last level time-like
+        y_pred : pandas.DataFrame with MultiIndex, last level time-like
+        non-time-like instanceso of y_true, y_pred must be identical
+        """
+        kwargsi = deepcopy(kwargs)
+        n_batches = len(y_true)
+        res = []
+        for i in range(n_batches):
+            if "y_train" in kwargs:
+                kwargsi["y_train"] = kwargs["y_train"][i]
+            if "y_pred_benchmark" in kwargs:
+                kwargsi["y_pred_benchmark"] = kwargs["y_pred_benchmark"][i]
+            resi = self._evaluate(y_true=y_true[i], y_pred=y_pred[i], **kwargsi)
+            if isinstance(resi, float):
+                resi = pd.Series(resi)
+            if self.multioutput == "raw_values":
+                assert isinstance(resi, np.ndarray)
+                df = pd.DataFrame(columns=y_true.X.columns)
+                df.loc[0] = resi
+                resi = df
+            res += [resi]
+        out_df = y_true.reconstruct(res)
+        if out_df.index.nlevels == y_true.X.index.nlevels:
+            out_df.index = out_df.index.droplevel(-1)
+
+        return out_df
+
+    def evaluate_by_index(self, y_true, y_pred, **kwargs):
+        """Return the metric evaluated at each time point.
+
+        Parameters
+        ----------
+        y_true : time series in aeon compatible data container format
+            Ground truth (correct) target values
+            y can be in one of the following formats:
+            Series type: pd.Series, pd.DataFrame, or np.ndarray (1D or 2D)
+            Panel type: pd.DataFrame with 2-level row MultiIndex,
+                3D np.ndarray, list of Series pd.DataFrame, or nested pd.DataFrame
+            Hierarchical type: pd.DataFrame with 3 or more level row MultiIndex
+        y_pred :time series in aeon compatible data container format
+            Forecasted values to evaluate
+            must be of same format as y_true, same indices and columns if indexed
+
+        Returns
+        -------
+        loss : pd.Series or pd.DataFrame
+            Calculated metric, by time point (default=jackknife pseudo-values).
+            pd.Series if self.multioutput="uniform_average" or array-like
+                index is equal to index of y_true
+                entry at index i is metric at time i, averaged over variables
+            pd.DataFrame if self.multioutput="raw_values"
+                index and columns equal to those of y_true
+                i,j-th entry is metric at time i, at variable j
+        """
+        multioutput = self.multioutput
+        multilevel = self.multilevel
+        # Input checks and conversions
+        y_true_inner, y_pred_inner, multioutput, multilevel, kwargs = self._check_ys(
+            y_true, y_pred, multioutput, multilevel, **kwargs
+        )
+        # pass to inner function
+        out_df = self._evaluate_by_index(y_true_inner, y_pred_inner, **kwargs)
+
+        return out_df
+
+    def _evaluate_by_index(self, y_true, y_pred, **kwargs):
+        """Return the metric evaluated at each time point.
+
+        private _evaluate_by_index containing core logic, called from evaluate_by_index
+
+        By default this uses _evaluate to find jackknifed pseudosamples.
+        This yields estimates for the metric at each of the time points.
+        Caution: this is only sensible for differentiable statistics,
+        i.e., not for medians, quantiles or median/quantile based statistics.
+
+        Parameters
+        ----------
+        y_true : time series in aeon compatible data container format
+            Ground truth (correct) target values
+            y can be in one of the following formats:
+            Series type: pd.Series, pd.DataFrame, or np.ndarray (1D or 2D)
+            Panel type: pd.DataFrame with 2-level row MultiIndex,
+                3D np.ndarray, list of Series pd.DataFrame, or nested pd.DataFrame
+            Hierarchical type: pd.DataFrame with 3 or more level row MultiIndex
+        y_pred :time series in aeon compatible data container format
+            Forecasted values to evaluate
+            must be of same format as y_true, same indices and columns if indexed
+
+        Returns
+        -------
+        loss : pd.Series or pd.DataFrame
+            Calculated metric, by time point (default=jackknife pseudo-values).
+            pd.Series if self.multioutput="uniform_average" or array-like
+                index is equal to index of y_true
+                entry at index i is metric at time i, averaged over variables
+            pd.DataFrame if self.multioutput="raw_values"
+                index and columns equal to those of y_true
+                i,j-th entry is metric at time i, at variable j
+        """
+        multioutput = self.multioutput
+        n = y_true.shape[0]
+        if multioutput == "raw_values":
+            out_series = pd.DataFrame(index=y_true.index, columns=y_true.columns)
+        else:
+            out_series = pd.Series(index=y_true.index)
+        try:
+            x_bar = self.evaluate(y_true, y_pred, **kwargs)
+            for i in range(n):
+                idx = y_true.index[i]
+                pseudovalue = n * x_bar - (n - 1) * self.evaluate(
+                    y_true.drop(idx),
+                    y_pred.drop(idx),
+                )
+                out_series.loc[idx] = pseudovalue
+            return out_series
+        except RecursionError:
+            RecursionError("Must implement one of _evaluate or _evaluate_by_index")
+
+    def _check_consistent_input(self, y_true, y_pred, multioutput, multilevel):
+        y_true_orig = y_true
+        y_pred_orig = y_pred
+
+        # unwrap y_true, y_pred, if wrapped in VectorizedDF
+        if isinstance(y_true, VectorizedDF):
+            y_true = y_true.X
+        if isinstance(y_pred, VectorizedDF):
+            y_pred = y_pred.X
+
+        # check row and column indices if y_true vs y_pred
+        same_rows = y_true.index.equals(y_pred.index)
+        same_row_num = len(y_true.index) == len(y_pred.index)
+        same_cols = y_true.columns.equals(y_pred.columns)
+        same_col_num = len(y_true.columns) == len(y_pred.columns)
+
+        if not same_row_num:
+            raise ValueError("y_pred and y_true do not have the same number of rows.")
+        if not same_col_num:
+            raise ValueError(
+                "y_pred and y_true do not have the same number of columns."
+            )
+
+        if not same_rows:
+            warn(
+                "y_pred and y_true do not have the same row index. "
+                "This may indicate incorrect objects passed to the metric. "
+                "Indices of y_true will be used for y_pred."
+            )
+            y_pred_orig = y_pred_orig.copy()
+            if isinstance(y_pred_orig, VectorizedDF):
+                y_pred_orig.X.index = y_true.index
+            else:
+                y_pred_orig.index = y_true.index
+        if not same_cols:
+            warn(
+                "y_pred and y_true do not have the same column index. "
+                "This may indicate incorrect objects passed to the metric. "
+                "Indices of y_true will be used for y_pred."
+            )
+            y_pred_orig = y_pred_orig.copy()
+            if isinstance(y_pred_orig, VectorizedDF):
+                y_pred_orig.X.columns = y_true.columns
+            else:
+                y_pred_orig.columns = y_true.columns
+        # check multioutput arg
+        # add this back when variance_weighted is supported
+        # ("raw_values", "uniform_average", "variance_weighted")
+        allowed_multioutput_str = ("raw_values", "uniform_average")
+
+        if isinstance(multioutput, str):
+            if multioutput not in allowed_multioutput_str:
+                raise ValueError(
+                    f"Allowed 'multioutput' values are {allowed_multioutput_str}, "
+                    f"but found multioutput={multioutput}"
+                )
+        else:
+            multioutput = check_array(multioutput, ensure_2d=False)
+            if len(y_pred.columns) != len(multioutput):
+                raise ValueError(
+                    "There must be equally many custom weights (%d) as outputs (%d)."
+                    % (len(multioutput), len(y_pred.columns))
+                )
+
+        # check multilevel arg
+        allowed_multilevel_str = (
+            "raw_values",
+            "uniform_average",
+            "uniform_average_time",
+        )
+
+        if not isinstance(multilevel, str):
+            raise ValueError(f"multilevel must be a str, but found {type(multilevel)}")
+        if multilevel not in allowed_multilevel_str:
+            raise ValueError(
+                f"Allowed 'multilevel' values are {allowed_multilevel_str}, "
+                f"but found multilevel={multilevel}"
+            )
+
+        return y_true_orig, y_pred_orig, multioutput, multilevel
+
+    def _check_ys(self, y_true, y_pred, multioutput, multilevel, **kwargs):
+        types = ["Series", "Panel", "Hierarchical"]
+        inner_types = ["pd.DataFrame", "pd-multiindex", "pd_multiindex_hier"]
+
+        def _coerce_to_df(y, var_name="y"):
+            valid, msg, metadata = check_is_scitype(
+                y, scitype=types, return_metadata=True, var_name=var_name
+            )
+            if not valid:
+                raise TypeError(msg)
+            y_inner = convert_to(y, to_type=inner_types)
+
+            type = metadata["scitype"]
+            ignore_index = multilevel == "uniform_average_time"
+            if type in ["Panel", "Hierarchical"] and not ignore_index:
+                y_inner = VectorizedDF(y_inner, is_scitype=type)
+            return y_inner
+
+        y_true = _coerce_to_df(y_true, var_name="y_true")
+        y_pred = _coerce_to_df(y_pred, var_name="y_pred")
+        if "y_train" in kwargs.keys():
+            kwargs["y_train"] = _coerce_to_df(kwargs["y_train"], var_name="y_train")
+        if "y_pred_benchmark" in kwargs.keys():
+            kwargs["y_pred_benchmark"] = _coerce_to_df(
+                kwargs["y_pred_benchmark"], var_name="y_pred_benchmark"
+            )
+
+        y_true, y_pred, multioutput, multilevel = self._check_consistent_input(
+            y_true, y_pred, multioutput, multilevel
+        )
+
+        return y_true, y_pred, multioutput, multilevel, kwargs
 
 
 class _BaseProbaForecastingErrorMetric(BaseForecastingErrorMetric):
diff --git a/aeon/performance_metrics/stats.py b/aeon/performance_metrics/stats.py
index 672a4ce9..2c0d3fc3 100644
--- a/aeon/performance_metrics/stats.py
+++ b/aeon/performance_metrics/stats.py
@@ -19,7 +19,7 @@ def check_friedman(ranks):
     Parameters
     ----------
     ranks : np.array
-      Rank of estimators on datasets, shape (n_estimators, n_datasets).
+        Rank of estimators on datasets, shape (n_estimators, n_datasets).
 
     Returns
     -------
@@ -92,8 +92,8 @@ def wilcoxon_test(results, labels, lower_better=False):
 
     Parameters
     ----------
-    results: np.array
-      results of estimators on datasets
+    results : np.array
+        Results of estimators on datasets
 
     lower_better : bool, default = False
         Indicates whether smaller is better for the results in scores. For example,
diff --git a/aeon/utils/mlflow_aeon.py b/aeon/utils/mlflow_aeon.py
index 0a2af06b..fe90f50a 100644
--- a/aeon/utils/mlflow_aeon.py
+++ b/aeon/utils/mlflow_aeon.py
@@ -48,6 +48,7 @@ import pickle
 import numpy as np
 import pandas as pd
 import yaml
+from deprecated.sphinx import deprecated
 
 import aeon
 from aeon import utils
@@ -84,6 +85,12 @@ SUPPORTED_SERIALIZATION_FORMATS = [
 _logger = logging.getLogger(__name__)
 
 
+# TODO: remove in v0.8.0
+@deprecated(
+    version="0.6.0",
+    reason="get_default_pip_requirements will be removed in v0.8.0.",
+    category=FutureWarning,
+)
 def get_default_pip_requirements(include_cloudpickle=False):
     """Create list of default pip requirements for MLflow Models.
 
@@ -103,6 +110,12 @@ def get_default_pip_requirements(include_cloudpickle=False):
     return pip_deps
 
 
+# TODO: remove in v0.8.0
+@deprecated(
+    version="0.6.0",
+    reason="get_default_conda_env will be removed in v0.8.0.",
+    category=FutureWarning,
+)
 def get_default_conda_env(include_cloudpickle=False):
     """Return default Conda environment for MLflow Models.
 
@@ -119,6 +132,12 @@ def get_default_conda_env(include_cloudpickle=False):
     )
 
 
+# TODO: remove in v0.8.0
+@deprecated(
+    version="0.6.0",
+    reason="save_model will be removed in v0.8.0.",
+    category=FutureWarning,
+)
 def save_model(
     estimator,
     path,
@@ -306,6 +325,12 @@ def save_model(
     _PythonEnv.current().to_yaml(os.path.join(path, _PYTHON_ENV_FILE_NAME))
 
 
+# TODO: remove in v0.8.0
+@deprecated(
+    version="0.6.0",
+    reason="log_model will be removed in v0.8.0.",
+    category=FutureWarning,
+)
 def log_model(
     estimator,
     artifact_path,
@@ -444,6 +469,12 @@ def log_model(
     )
 
 
+# TODO: remove in v0.8.0
+@deprecated(
+    version="0.6.0",
+    reason="load_model will be removed in v0.8.0.",
+    category=FutureWarning,
+)
 def load_model(model_uri, dst_path=None):
     """
     Load a aeon model from a local file or a run.
diff --git a/aeon/utils/validation/forecasting.py b/aeon/utils/validation/forecasting.py
index b674bc06..e22accb3 100644
--- a/aeon/utils/validation/forecasting.py
+++ b/aeon/utils/validation/forecasting.py
@@ -414,20 +414,10 @@ def check_scoring(scoring, allow_y_pred_benchmark=False):
     NotImplementedError
         if metric requires y_pred_benchmark to be passed
     """
-    # Note symmetric=True is default arg for MeanAbsolutePercentageError
-    from aeon.performance_metrics.forecasting import MeanAbsolutePercentageError
+    from aeon.performance_metrics.forecasting import mean_absolute_percentage_error
 
     if scoring is None:
-        return MeanAbsolutePercentageError()
-
-    scoring_req_bench = scoring.get_class_tag("requires-y-pred-benchmark", False)
-
-    if scoring_req_bench and not allow_y_pred_benchmark:
-        msg = """Scoring requiring benchmark forecasts (y_pred_benchmark) are not
-                 fully supported yet. Please use a performance metric that does not
-                 require y_pred_benchmark as a keyword argument in its call signature.
-              """
-        raise NotImplementedError(msg)
+        return mean_absolute_percentage_error
 
     if not callable(scoring):
         raise TypeError("`scoring` must be a callable object")
diff --git a/aeon/visualisation/results/_significance.py b/aeon/visualisation/results/_significance.py
index d524fafe..fbae607d 100644
--- a/aeon/visualisation/results/_significance.py
+++ b/aeon/visualisation/results/_significance.py
@@ -1,3 +1,11 @@
+"""Function to compute and plot significance."""
+
+__author__ = ["dguijo", "TonyBagnall"]
+
+__all__ = [
+    "plot_significance",
+]
+
 import warnings
 
 import numpy as np
@@ -19,13 +27,18 @@ def plot_significance(
     return_p_values=False,
 ):
     """
-    Plot that allow the case where cliques can be deceiving.
+    Plot similar to CDDs, but allows the case where cliques can be deceiving.
 
     It is a visual representation of the results of a statistical test to compare the
     performance of different classifiers. The plot is based on the work of Demsar [1]_.
     The plot shows the average rank of each classifier and the average score of each
     method. The plot also shows the cliques of classifiers that are not significantly
-    different from each other.
+    different from each other. The main difference against CDDs is that this plot
+    allows the case where cliques can be deceiving, i.e. there is a gap where a method
+    is significantly different from the rest of the clique. In the CDDs, the clique
+    stop when there is a significant difference between the methods. In this plot,
+    the clique continues until the end of the list of methods, showing a gap for the
+    method with significant differences.
 
     Parameters
     ----------
@@ -52,13 +65,13 @@ def plot_significance(
 
     Returns
     -------
-        fig : matplotlib.figure
-            Figure created.
-        ax : matplotlib.axes
-            Axes of the figure.
-        p_values : np.ndarray (optional)
-            if return_p_values is True, returns a (n_estimators, n_estimators) matrix of
-            unadjusted p values for the pairwise Wilcoxon sign rank test.
+    fig : matplotlib.figure
+        Figure created.
+    ax : matplotlib.axes
+        Axes of the figure.
+    p_values : np.ndarray (optional)
+        if return_p_values is True, returns a (n_estimators, n_estimators) matrix of
+        unadjusted p values for the pairwise Wilcoxon sign rank test.
 
     References
     ----------
diff --git a/docs/api_reference.md b/docs/api_reference.md
index 2488f355..b022f054 100644
--- a/docs/api_reference.md
+++ b/docs/api_reference.md
@@ -11,13 +11,13 @@ and [examples](examples.md) page.
 ```{toctree}
 :maxdepth: 1
 
-api_reference/annotation
+api_reference/anomaly_detection
 api_reference/base
 api_reference/benchmarking
 api_reference/classification
 api_reference/clustering
+api_reference/data_format
 api_reference/datasets
-api_reference/deployment
 api_reference/distances
 api_reference/exceptions
 api_reference/forecasting
@@ -28,5 +28,5 @@ api_reference/segmentation
 api_reference/similarity_search
 api_reference/transformations
 api_reference/utils
-api_reference/file_specifications/ts
+api_reference/visualisation
 ```
diff --git a/docs/api_reference/file_specifications/ts.rst b/docs/api_reference/data_format.rst
similarity index 100%
rename from docs/api_reference/file_specifications/ts.rst
rename to docs/api_reference/data_format.rst
diff --git a/docs/api_reference/performance_metrics.rst b/docs/api_reference/performance_metrics.rst
index 324e6afa..18aeecf0 100644
--- a/docs/api_reference/performance_metrics.rst
+++ b/docs/api_reference/performance_metrics.rst
@@ -15,38 +15,6 @@ Forecasting
 
 .. currentmodule:: aeon.performance_metrics.forecasting
 
-Classes
-~~~~~~~
-
-.. autosummary::
-    :toctree: auto_generated/
-    :template: class_with_call.rst
-
-    MeanAbsoluteScaledError
-    MedianAbsoluteScaledError
-    MeanSquaredScaledError
-    MedianSquaredScaledError
-    MeanAbsoluteError
-    MeanSquaredError
-    MedianAbsoluteError
-    MedianSquaredError
-    GeometricMeanAbsoluteError
-    GeometricMeanSquaredError
-    MeanAbsolutePercentageError
-    MedianAbsolutePercentageError
-    MeanSquaredPercentageError
-    MedianSquaredPercentageError
-    MeanRelativeAbsoluteError
-    MedianRelativeAbsoluteError
-    GeometricMeanRelativeAbsoluteError
-    GeometricMeanRelativeSquaredError
-    MeanAsymmetricError
-    MeanLinexError
-    RelativeLoss
-
-Functions
-~~~~~~~~~
-
 .. autosummary::
     :toctree: auto_generated/
     :template: function.rst
diff --git a/docs/api_reference/utils.rst b/docs/api_reference/utils.rst
index 3a828441..d80626d4 100644
--- a/docs/api_reference/utils.rst
+++ b/docs/api_reference/utils.rst
@@ -5,40 +5,13 @@ Utility functions
 
 ``aeon`` has a number of modules dedicated to utilities:
 
-* :mod:`aeon.datatypes`, which contains utilities for data format checks and conversion.
 * :mod:`aeon.pipeline`, which contains generics for pipeline construction.
 * :mod:`aeon.registry`, which contains utilities for estimator and tag search.
 * :mod:`aeon.utils`, which contains generic utility functions.
 
 
-Data Format Checking and Conversion
------------------------------------
-
-:mod:`aeon.datatypes`
-
-.. automodule:: aeon.datatypes
-    :no-members:
-    :no-inherited-members:
-
-.. currentmodule:: aeon.datatypes
-
-.. autosummary::
-    :toctree: auto_generated/
-    :template: function.rst
-
-    convert_to
-    convert
-    check_raise
-    check_is_mtype
-    check_is_scitype
-    mtype
-    scitype
-    mtype_to_scitype
-    scitype_to_mtype
-
-
-Pipeline construction generics
-------------------------------
+Pipeline construction
+---------------------
 
 :mod:`aeon.pipeline`
 
@@ -75,6 +48,7 @@ Estimator Search and Retrieval, Estimator Tags
     all_tags
     check_tag_is_valid
 
+
 Estimator Validity Checking
 ---------------------------
 
@@ -91,3 +65,20 @@ Estimator Validity Checking
     :template: function.rst
 
     check_estimator
+
+Data Validation Tools
+---------------------
+
+:mod:`aeon.utils`
+
+.. automodule:: aeon.utils.validation
+    :no-members:
+    :no-inherited-members:
+
+.. currentmodule:: aeon.utils.estimator_checks
+
+.. autosummary::
+    :toctree: auto_generated/
+    :template: function.rst
+
+    convert_collection
diff --git a/examples/benchmarking/forecasting.ipynb b/examples/benchmarking/forecasting.ipynb
index 76bd0d91..7be498c4 100644
--- a/examples/benchmarking/forecasting.ipynb
+++ b/examples/benchmarking/forecasting.ipynb
@@ -15,7 +15,7 @@
     "from aeon.datasets import load_airline\n",
     "from aeon.forecasting.model_selection import ExpandingWindowSplitter\n",
     "from aeon.forecasting.naive import NaiveForecaster\n",
-    "from aeon.performance_metrics.forecasting import MeanSquaredPercentageError"
+    "from aeon.performance_metrics.forecasting import mean_squared_percentage_error"
    ]
   },
   {
@@ -105,7 +105,8 @@
    "metadata": {},
    "source": [
     "#### Specify performance metric(s)\n",
-    "Define performance metrics on which to compare estimators, using standard `aeon` objects."
+    "Define performance metrics on which to compare estimators, using standard `aeon`\n",
+    "functions."
    ]
   },
   {
@@ -115,7 +116,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "scorers = [MeanSquaredPercentageError()]"
+    "scorers = [mean_squared_percentage_error]"
    ]
   },
   {
diff --git a/examples/forecasting/forecasting.ipynb b/examples/forecasting/forecasting.ipynb
index 4eaed090..ab809e62 100644
--- a/examples/forecasting/forecasting.ipynb
+++ b/examples/forecasting/forecasting.ipynb
@@ -1293,7 +1293,8 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "To predict full predictive distributions, `predict_proba` can be used."
+    "To predict full predictive distributions, `predict_proba` can be used.\n",
+    "As this returns `tensorflow` `Distribution` objects, the deep learning dependency set `dl` of `aeon` (which includes `tensorflow` and `tensorflow-probability` dependencies) must be installed."
    ]
   },
   {
@@ -1308,7 +1309,7 @@
    "outputs": [
     {
      "data": {
-      "text/plain": "<scipy.stats._distn_infrastructure.rv_frozen at 0x1a6d4938ac0>"
+      "text/plain": "<tfp.distributions.Normal 'Normal' batch_shape=[12, 1] event_shape=[] dtype=float32>"
      },
      "execution_count": 33,
      "metadata": {},
@@ -1362,8 +1363,8 @@
    "outputs": [
     {
      "data": {
-      "text/plain": "                                  passengers\nl2_agg    l1_agg    timepoints              \nl2_node01 l1_node03 1949-01      1578.780000\n                    1949-02      1662.780000\n                    1949-03      1858.780000\n                    1949-04      1816.780000\n                    1949-05      1704.780000\n...                                      ...\nl2_node03 l1_node06 1960-08     11718.065993\n                    1960-09      9750.491696\n                    1960-10      8814.676284\n                    1960-11      7411.341802\n                    1960-12      8239.936597\n\n[864 rows x 1 columns]",
-      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th></th>\n      <th></th>\n      <th>passengers</th>\n    </tr>\n    <tr>\n      <th>l2_agg</th>\n      <th>l1_agg</th>\n      <th>timepoints</th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th rowspan=\"5\" valign=\"top\">l2_node01</th>\n      <th rowspan=\"5\" valign=\"top\">l1_node03</th>\n      <th>1949-01</th>\n      <td>1578.780000</td>\n    </tr>\n    <tr>\n      <th>1949-02</th>\n      <td>1662.780000</td>\n    </tr>\n    <tr>\n      <th>1949-03</th>\n      <td>1858.780000</td>\n    </tr>\n    <tr>\n      <th>1949-04</th>\n      <td>1816.780000</td>\n    </tr>\n    <tr>\n      <th>1949-05</th>\n      <td>1704.780000</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <th>...</th>\n      <th>...</th>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th rowspan=\"5\" valign=\"top\">l2_node03</th>\n      <th rowspan=\"5\" valign=\"top\">l1_node06</th>\n      <th>1960-08</th>\n      <td>11718.065993</td>\n    </tr>\n    <tr>\n      <th>1960-09</th>\n      <td>9750.491696</td>\n    </tr>\n    <tr>\n      <th>1960-10</th>\n      <td>8814.676284</td>\n    </tr>\n    <tr>\n      <th>1960-11</th>\n      <td>7411.341802</td>\n    </tr>\n    <tr>\n      <th>1960-12</th>\n      <td>8239.936597</td>\n    </tr>\n  </tbody>\n</table>\n<p>864 rows × 1 columns</p>\n</div>"
+      "text/plain": "                                 passengers\nl2_agg    l1_agg    timepoints             \nl2_node01 l1_node02 1949-01       52.943320\n                    1949-02       54.874511\n                    1949-03       59.379565\n                    1949-04       58.414313\n                    1949-05       55.840000\n...                                     ...\nl2_node03 l1_node05 1960-08     2221.886565\n                    1960-09     1801.134311\n                    1960-10     1606.045386\n                    1960-11     1320.202973\n                    1960-12     1487.967083\n\n[864 rows x 1 columns]",
+      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th></th>\n      <th></th>\n      <th>passengers</th>\n    </tr>\n    <tr>\n      <th>l2_agg</th>\n      <th>l1_agg</th>\n      <th>timepoints</th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th rowspan=\"5\" valign=\"top\">l2_node01</th>\n      <th rowspan=\"5\" valign=\"top\">l1_node02</th>\n      <th>1949-01</th>\n      <td>52.943320</td>\n    </tr>\n    <tr>\n      <th>1949-02</th>\n      <td>54.874511</td>\n    </tr>\n    <tr>\n      <th>1949-03</th>\n      <td>59.379565</td>\n    </tr>\n    <tr>\n      <th>1949-04</th>\n      <td>58.414313</td>\n    </tr>\n    <tr>\n      <th>1949-05</th>\n      <td>55.840000</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <th>...</th>\n      <th>...</th>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th rowspan=\"5\" valign=\"top\">l2_node03</th>\n      <th rowspan=\"5\" valign=\"top\">l1_node05</th>\n      <th>1960-08</th>\n      <td>2221.886565</td>\n    </tr>\n    <tr>\n      <th>1960-09</th>\n      <td>1801.134311</td>\n    </tr>\n    <tr>\n      <th>1960-10</th>\n      <td>1606.045386</td>\n    </tr>\n    <tr>\n      <th>1960-11</th>\n      <td>1320.202973</td>\n    </tr>\n    <tr>\n      <th>1960-12</th>\n      <td>1487.967083</td>\n    </tr>\n  </tbody>\n</table>\n<p>864 rows × 1 columns</p>\n</div>"
      },
      "execution_count": 34,
      "metadata": {},
@@ -1398,8 +1399,8 @@
    "outputs": [
     {
      "data": {
-      "text/plain": "                                 passengers\nl2_agg    l1_agg    timepoints             \nl2_node01 l1_node03 1961-01     5982.426359\n                    1961-02     5908.776162\n                    1961-03     5837.733689\n          l1_node04 1961-01     1570.438463\n                    1961-02     1551.221984\n                    1961-03     1532.686180\nl2_node02 l1_node02 1961-01     4536.083657\n                    1961-02     4478.308154\n                    1961-03     4422.623524\n          l1_node05 1961-01     6139.831381\n                    1961-02     6058.285845\n                    1961-03     5979.762685\nl2_node03 l1_node01 1961-01      426.544850\n                    1961-02      421.282983\n                    1961-03      416.207550\n          l1_node06 1961-01     8131.244318\n                    1961-02     8026.509695\n                    1961-03     7925.588623",
-      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th></th>\n      <th></th>\n      <th>passengers</th>\n    </tr>\n    <tr>\n      <th>l2_agg</th>\n      <th>l1_agg</th>\n      <th>timepoints</th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th rowspan=\"6\" valign=\"top\">l2_node01</th>\n      <th rowspan=\"3\" valign=\"top\">l1_node03</th>\n      <th>1961-01</th>\n      <td>5982.426359</td>\n    </tr>\n    <tr>\n      <th>1961-02</th>\n      <td>5908.776162</td>\n    </tr>\n    <tr>\n      <th>1961-03</th>\n      <td>5837.733689</td>\n    </tr>\n    <tr>\n      <th rowspan=\"3\" valign=\"top\">l1_node04</th>\n      <th>1961-01</th>\n      <td>1570.438463</td>\n    </tr>\n    <tr>\n      <th>1961-02</th>\n      <td>1551.221984</td>\n    </tr>\n    <tr>\n      <th>1961-03</th>\n      <td>1532.686180</td>\n    </tr>\n    <tr>\n      <th rowspan=\"6\" valign=\"top\">l2_node02</th>\n      <th rowspan=\"3\" valign=\"top\">l1_node02</th>\n      <th>1961-01</th>\n      <td>4536.083657</td>\n    </tr>\n    <tr>\n      <th>1961-02</th>\n      <td>4478.308154</td>\n    </tr>\n    <tr>\n      <th>1961-03</th>\n      <td>4422.623524</td>\n    </tr>\n    <tr>\n      <th rowspan=\"3\" valign=\"top\">l1_node05</th>\n      <th>1961-01</th>\n      <td>6139.831381</td>\n    </tr>\n    <tr>\n      <th>1961-02</th>\n      <td>6058.285845</td>\n    </tr>\n    <tr>\n      <th>1961-03</th>\n      <td>5979.762685</td>\n    </tr>\n    <tr>\n      <th rowspan=\"6\" valign=\"top\">l2_node03</th>\n      <th rowspan=\"3\" valign=\"top\">l1_node01</th>\n      <th>1961-01</th>\n      <td>426.544850</td>\n    </tr>\n    <tr>\n      <th>1961-02</th>\n      <td>421.282983</td>\n    </tr>\n    <tr>\n      <th>1961-03</th>\n      <td>416.207550</td>\n    </tr>\n    <tr>\n      <th rowspan=\"3\" valign=\"top\">l1_node06</th>\n      <th>1961-01</th>\n      <td>8131.244318</td>\n    </tr>\n    <tr>\n      <th>1961-02</th>\n      <td>8026.509695</td>\n    </tr>\n    <tr>\n      <th>1961-03</th>\n      <td>7925.588623</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
+      "text/plain": "                                 passengers\nl2_agg    l1_agg    timepoints             \nl2_node01 l1_node02 1961-01      154.000381\n                    1961-02      152.311358\n                    1961-03      150.682121\n          l1_node03 1961-01     2805.041069\n                    1961-02     2770.577861\n                    1961-03     2737.335536\nl2_node02 l1_node01 1961-01      426.544850\n                    1961-02      421.282983\n                    1961-03      416.207550\n          l1_node04 1961-01      892.307468\n                    1961-02      879.432170\n                    1961-03      867.062017\n          l1_node06 1961-01     3237.664605\n                    1961-02     3192.489057\n                    1961-03     3149.040796\nl2_node03 l1_node05 1961-01     1465.123534\n                    1961-02     1443.194107\n                    1961-03     1422.142219",
+      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th></th>\n      <th></th>\n      <th>passengers</th>\n    </tr>\n    <tr>\n      <th>l2_agg</th>\n      <th>l1_agg</th>\n      <th>timepoints</th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th rowspan=\"6\" valign=\"top\">l2_node01</th>\n      <th rowspan=\"3\" valign=\"top\">l1_node02</th>\n      <th>1961-01</th>\n      <td>154.000381</td>\n    </tr>\n    <tr>\n      <th>1961-02</th>\n      <td>152.311358</td>\n    </tr>\n    <tr>\n      <th>1961-03</th>\n      <td>150.682121</td>\n    </tr>\n    <tr>\n      <th rowspan=\"3\" valign=\"top\">l1_node03</th>\n      <th>1961-01</th>\n      <td>2805.041069</td>\n    </tr>\n    <tr>\n      <th>1961-02</th>\n      <td>2770.577861</td>\n    </tr>\n    <tr>\n      <th>1961-03</th>\n      <td>2737.335536</td>\n    </tr>\n    <tr>\n      <th rowspan=\"9\" valign=\"top\">l2_node02</th>\n      <th rowspan=\"3\" valign=\"top\">l1_node01</th>\n      <th>1961-01</th>\n      <td>426.544850</td>\n    </tr>\n    <tr>\n      <th>1961-02</th>\n      <td>421.282983</td>\n    </tr>\n    <tr>\n      <th>1961-03</th>\n      <td>416.207550</td>\n    </tr>\n    <tr>\n      <th rowspan=\"3\" valign=\"top\">l1_node04</th>\n      <th>1961-01</th>\n      <td>892.307468</td>\n    </tr>\n    <tr>\n      <th>1961-02</th>\n      <td>879.432170</td>\n    </tr>\n    <tr>\n      <th>1961-03</th>\n      <td>867.062017</td>\n    </tr>\n    <tr>\n      <th rowspan=\"3\" valign=\"top\">l1_node06</th>\n      <th>1961-01</th>\n      <td>3237.664605</td>\n    </tr>\n    <tr>\n      <th>1961-02</th>\n      <td>3192.489057</td>\n    </tr>\n    <tr>\n      <th>1961-03</th>\n      <td>3149.040796</td>\n    </tr>\n    <tr>\n      <th rowspan=\"3\" valign=\"top\">l2_node03</th>\n      <th rowspan=\"3\" valign=\"top\">l1_node05</th>\n      <th>1961-01</th>\n      <td>1465.123534</td>\n    </tr>\n    <tr>\n      <th>1961-02</th>\n      <td>1443.194107</td>\n    </tr>\n    <tr>\n      <th>1961-03</th>\n      <td>1422.142219</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
      },
      "execution_count": 35,
      "metadata": {},
@@ -1436,8 +1437,8 @@
    "outputs": [
     {
      "data": {
-      "text/plain": "                    forecasters\nl2_agg    l1_agg               \nl2_node01 l1_node03     ARIMA()\n          l1_node04     ARIMA()\nl2_node02 l1_node02     ARIMA()\n          l1_node05     ARIMA()\nl2_node03 l1_node01     ARIMA()\n          l1_node06     ARIMA()",
-      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th></th>\n      <th>forecasters</th>\n    </tr>\n    <tr>\n      <th>l2_agg</th>\n      <th>l1_agg</th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th rowspan=\"2\" valign=\"top\">l2_node01</th>\n      <th>l1_node03</th>\n      <td>ARIMA()</td>\n    </tr>\n    <tr>\n      <th>l1_node04</th>\n      <td>ARIMA()</td>\n    </tr>\n    <tr>\n      <th rowspan=\"2\" valign=\"top\">l2_node02</th>\n      <th>l1_node02</th>\n      <td>ARIMA()</td>\n    </tr>\n    <tr>\n      <th>l1_node05</th>\n      <td>ARIMA()</td>\n    </tr>\n    <tr>\n      <th rowspan=\"2\" valign=\"top\">l2_node03</th>\n      <th>l1_node01</th>\n      <td>ARIMA()</td>\n    </tr>\n    <tr>\n      <th>l1_node06</th>\n      <td>ARIMA()</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
+      "text/plain": "                    forecasters\nl2_agg    l1_agg               \nl2_node01 l1_node02     ARIMA()\n          l1_node03     ARIMA()\nl2_node02 l1_node01     ARIMA()\n          l1_node04     ARIMA()\n          l1_node06     ARIMA()\nl2_node03 l1_node05     ARIMA()",
+      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th></th>\n      <th>forecasters</th>\n    </tr>\n    <tr>\n      <th>l2_agg</th>\n      <th>l1_agg</th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th rowspan=\"2\" valign=\"top\">l2_node01</th>\n      <th>l1_node02</th>\n      <td>ARIMA()</td>\n    </tr>\n    <tr>\n      <th>l1_node03</th>\n      <td>ARIMA()</td>\n    </tr>\n    <tr>\n      <th rowspan=\"3\" valign=\"top\">l2_node02</th>\n      <th>l1_node01</th>\n      <td>ARIMA()</td>\n    </tr>\n    <tr>\n      <th>l1_node04</th>\n      <td>ARIMA()</td>\n    </tr>\n    <tr>\n      <th>l1_node06</th>\n      <td>ARIMA()</td>\n    </tr>\n    <tr>\n      <th>l2_node03</th>\n      <th>l1_node05</th>\n      <td>ARIMA()</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
      },
      "execution_count": 36,
      "metadata": {},
@@ -1457,24 +1458,17 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 37,
+   "execution_count": null,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:22.651879800Z",
      "start_time": "2023-08-25T09:25:14.218928600Z"
+    },
+    "pycharm": {
+     "is_executing": true
     }
    },
-   "outputs": [
-    {
-     "data": {
-      "text/plain": "                c0       c1\nh0   h1                    \nh0_0 h1_0  ARIMA()  ARIMA()\n     h1_1  ARIMA()  ARIMA()\n     h1_2  ARIMA()  ARIMA()\n     h1_3  ARIMA()  ARIMA()\nh0_1 h1_0  ARIMA()  ARIMA()\n     h1_1  ARIMA()  ARIMA()\n     h1_2  ARIMA()  ARIMA()\n     h1_3  ARIMA()  ARIMA()",
-      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th></th>\n      <th>c0</th>\n      <th>c1</th>\n    </tr>\n    <tr>\n      <th>h0</th>\n      <th>h1</th>\n      <th></th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th rowspan=\"4\" valign=\"top\">h0_0</th>\n      <th>h1_0</th>\n      <td>ARIMA()</td>\n      <td>ARIMA()</td>\n    </tr>\n    <tr>\n      <th>h1_1</th>\n      <td>ARIMA()</td>\n      <td>ARIMA()</td>\n    </tr>\n    <tr>\n      <th>h1_2</th>\n      <td>ARIMA()</td>\n      <td>ARIMA()</td>\n    </tr>\n    <tr>\n      <th>h1_3</th>\n      <td>ARIMA()</td>\n      <td>ARIMA()</td>\n    </tr>\n    <tr>\n      <th rowspan=\"4\" valign=\"top\">h0_1</th>\n      <th>h1_0</th>\n      <td>ARIMA()</td>\n      <td>ARIMA()</td>\n    </tr>\n    <tr>\n      <th>h1_1</th>\n      <td>ARIMA()</td>\n      <td>ARIMA()</td>\n    </tr>\n    <tr>\n      <th>h1_2</th>\n      <td>ARIMA()</td>\n      <td>ARIMA()</td>\n    </tr>\n    <tr>\n      <th>h1_3</th>\n      <td>ARIMA()</td>\n      <td>ARIMA()</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
-     },
-     "execution_count": 37,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
     "from aeon.forecasting.arima import ARIMA\n",
     "from aeon.testing.utils.data_gen import _make_hierarchical\n",
@@ -1491,14 +1485,15 @@
   },
   {
    "cell_type": "markdown",
-   "metadata": {},
    "source": [
     "Further details on hierarchical forecasting, including reduction, aggregation, reconciliation, are presented in the \"hierarchical forecasting\" tutorial."
-   ]
+   ],
+   "metadata": {
+    "collapsed": false
+   }
   },
   {
    "cell_type": "markdown",
-   "metadata": {},
    "source": [
     "### 1.3 Basic evaluation workflow - evaluating a batch of forecasts against ground truth observations\n",
     "\n",
@@ -1515,61 +1510,56 @@
     "NOTE: Step 5 (testing) is currently not supported in `aeon`, but is on the development roadmap. For the time being, it is advised to use custom implementations of appropriate methods (e.g., Diebold-Mariano test; stationary confidence intervals).\n",
     "\n",
     "NOTE: Note that this evaluation set-up determines how well a given algorithm would have performed on past data. Results are only insofar representative as future performance can be assumed to mirror past performance. This can be argued under certain assumptions (e.g., stationarity), but will in general be false. Monitoring of forecasting performance is hence advised in case an algorithm is applied multiple times."
-   ]
+   ],
+   "metadata": {
+    "collapsed": false
+   }
   },
   {
    "cell_type": "markdown",
-   "metadata": {},
    "source": [
     "**Example:** In the example, we will us the same airline data as in Section 1.2. But, instead of predicting the next 3 years, we hold out the last 3 years of the airline data (below: `y_test`), and see how the forecaster would have performed three years ago, when asked to forecast the most recent 3 years (below: `y_pred`), from the years before (below: `y_train`). \"how\" is measured by a quantitative performance metric (below: `mean_absolute_percentage_error`). This is then considered as an indication of how well the forecaster would perform in the coming 3 years (what was done in Section 1.2). This may or may not be a stretch depending on statistical assumptions and data properties (caution: it often is a stretch - past performance is in general not indicative of future performance)."
-   ]
+   ],
+   "metadata": {
+    "collapsed": false
+   }
   },
   {
    "cell_type": "markdown",
-   "metadata": {},
    "source": [
     "#### Step 1 - Splitting a historical data set in to a temporal train and test batch"
-   ]
+   ],
+   "metadata": {
+    "collapsed": false
+   }
   },
   {
    "cell_type": "code",
    "execution_count": 38,
-   "metadata": {
-    "ExecuteTime": {
-     "end_time": "2023-08-25T09:25:22.652909500Z",
-     "start_time": "2023-08-25T09:25:14.754495800Z"
-    }
-   },
    "outputs": [],
    "source": [
     "from aeon.forecasting.model_selection import temporal_train_test_split"
-   ]
+   ],
+   "metadata": {
+    "collapsed": false
+   }
   },
   {
    "cell_type": "code",
    "execution_count": 39,
-   "metadata": {
-    "ExecuteTime": {
-     "end_time": "2023-08-25T09:25:22.652909500Z",
-     "start_time": "2023-08-25T09:25:14.759482700Z"
-    }
-   },
    "outputs": [],
    "source": [
     "y = load_airline()\n",
     "y_train, y_test = temporal_train_test_split(y, test_size=36)\n",
     "# we will try to forecast y_test from y_train"
-   ]
+   ],
+   "metadata": {
+    "collapsed": false
+   }
   },
   {
    "cell_type": "code",
    "execution_count": 40,
-   "metadata": {
-    "ExecuteTime": {
-     "end_time": "2023-08-25T09:25:22.710722400Z",
-     "start_time": "2023-08-25T09:25:14.769456500Z"
-    }
-   },
    "outputs": [
     {
      "name": "stdout",
@@ -1591,16 +1581,21 @@
     "# plotting for illustration\n",
     "plot_series(y_train, y_test, labels=[\"y_train\", \"y_test\"])\n",
     "print(y_train.shape[0], y_test.shape[0])"
-   ]
+   ],
+   "metadata": {
+    "collapsed": false
+   }
   },
   {
    "cell_type": "markdown",
-   "metadata": {},
    "source": [
     "#### Step 2 - Making forecasts for y_test from y_train\n",
     "\n",
     "This is almost verbatim the workflow in Section 1.2, using `y_train` to predict the indices of `y_test`."
-   ]
+   ],
+   "metadata": {
+    "collapsed": false
+   }
   },
   {
    "cell_type": "code",
@@ -1662,12 +1657,11 @@
    "source": [
     "#### Steps 3 and 4 - Specifying a forecasting metric, evaluating on the test set\n",
     "\n",
-    "The next step is to specify a forecasting metric. These are functions that return a number when input with prediction and actual series. They are different from `sklearn` metrics in that they accept series with indices rather than `np.array`s. Forecasting metrics can be invoked in two ways:\n",
-    "\n",
-    "* using the lean function interface, e.g., `mean_absolute_percentage_error` which is a python function `(y_true : pd.Series, y_pred : pd.Series) -> float`\n",
-    "* using the composable class interface, e.g., `MeanAbsolutePercentageError`, which is a python class, callable with the same signature\n",
-    "\n",
-    "Casual users may opt to use the function interface. The class interface supports advanced use cases, such as parameter modification, custom metric composition, tuning over metric parameters (not covered in this tutorial)"
+    "The next step is to specify a forecasting metric. These are functions that return a\n",
+    "number when input with prediction and actual series. They are different from\n",
+    "`sklearn` metrics in that they can accept series with indices rather than `np\n",
+    ".ndarray`. Forecasting metrics can be invoked using the functions in `aeon.benchmarking\n",
+    ".forecasting` e.g.,`mean_absolute_percentage_error` or `mean_squared_error`.\n"
    ]
   },
   {
@@ -1710,98 +1704,13 @@
     "#       the order matters for most metrics in general"
    ]
   },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "To properly interpret numbers like this, it is useful to understand properties of the metric in question (e.g., lower is better), and to compare against suitable baselines and contender algorithms (see step 5)."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 45,
-   "metadata": {
-    "ExecuteTime": {
-     "end_time": "2023-08-25T09:25:23.033861900Z",
-     "start_time": "2023-08-25T09:25:15.387803200Z"
-    }
-   },
-   "outputs": [],
-   "source": [
-    "from aeon.performance_metrics.forecasting import MeanAbsolutePercentageError"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 46,
-   "metadata": {
-    "ExecuteTime": {
-     "end_time": "2023-08-25T09:25:23.033861900Z",
-     "start_time": "2023-08-25T09:25:15.394784500Z"
-    }
-   },
-   "outputs": [
-    {
-     "data": {
-      "text/plain": "True"
-     },
-     "execution_count": 46,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "# option 2: using the composable class interface\n",
-    "mape = MeanAbsolutePercentageError(symmetric=False)\n",
-    "# the class interface allows to easily construct variants of the MAPE\n",
-    "#  e.g., the non-symmetric verion\n",
-    "# it also allows for inspection of metric properties\n",
-    "#  e.g., are higher values better (answer: no)?\n",
-    "mape.get_tag(\"lower_is_better\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 47,
-   "metadata": {
-    "ExecuteTime": {
-     "end_time": "2023-08-25T09:25:23.033861900Z",
-     "start_time": "2023-08-25T09:25:15.399771200Z"
-    }
-   },
-   "outputs": [
-    {
-     "data": {
-      "text/plain": "0.13189432350948402"
-     },
-     "execution_count": 47,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "# evaluation works exactly like in option 2, but with the instantiated object\n",
-    "mape(y_test, y_pred)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "NOTE: Some metrics, such as `mean_absolute_scaled_error`, also require the training set for evaluation. In this case, the training set should be passed as a `y_train` argument. Refer to the API reference on individual metrics.\n",
-    "\n",
-    "NOTE: The workflow is the same for forecasters that make use of exogeneous data - no `X` is passed to the metrics."
-   ]
-  },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "#### Step 5 - Testing performance against benchmarks\n",
     "\n",
-    "In general, forecast performances should be quantitatively tested against benchmark performances.\n",
-    "\n",
-    "Currently (`aeon` v0.12.x), this is a roadmap development item. Contributions are very welcome."
+    "In general, forecast performances should be quantitatively tested against benchmark performances.\n"
    ]
   },
   {
@@ -1816,7 +1725,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 48,
+   "execution_count": 45,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:23.033861900Z",
@@ -1834,7 +1743,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 49,
+   "execution_count": 46,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:23.034859300Z",
@@ -1846,7 +1755,7 @@
      "data": {
       "text/plain": "0.13189432350948402"
      },
-     "execution_count": 49,
+     "execution_count": 46,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -1882,7 +1791,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 50,
+   "execution_count": 47,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:23.035264300Z",
@@ -1900,7 +1809,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 51,
+   "execution_count": 48,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:23.044833Z",
@@ -1912,7 +1821,7 @@
      "data": {
       "text/plain": "0.13189432350948402"
      },
-     "execution_count": 51,
+     "execution_count": 48,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -1973,7 +1882,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 52,
+   "execution_count": 49,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:23.044833Z",
@@ -1989,7 +1898,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 53,
+   "execution_count": 50,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:23.044833Z",
@@ -2005,7 +1914,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 54,
+   "execution_count": 51,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:23.310476100Z",
@@ -2034,7 +1943,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 55,
+   "execution_count": 52,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:23.311473700Z",
@@ -2046,7 +1955,7 @@
      "data": {
       "text/plain": "(<Figure size 1600x400 with 1 Axes>,\n <Axes: ylabel='Number of airline passengers'>)"
      },
-     "execution_count": 55,
+     "execution_count": 52,
      "metadata": {},
      "output_type": "execute_result"
     },
@@ -2066,7 +1975,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 56,
+   "execution_count": 53,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:23.311473700Z",
@@ -2089,7 +1998,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 57,
+   "execution_count": 54,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:23.311473700Z",
@@ -2101,7 +2010,7 @@
      "data": {
       "text/plain": "1958-02    341.515812\n1958-03    392.849891\n1958-04    378.521584\n1958-05    375.662009\n1958-06    426.011081\n1958-07    470.577777\n1958-08    467.110706\n1958-09    414.461454\n1958-10    360.966279\n1958-11    315.212373\n1958-12    357.909015\n1959-01    363.046383\nFreq: M, dtype: float64"
      },
-     "execution_count": 57,
+     "execution_count": 54,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -2114,7 +2023,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 58,
+   "execution_count": 55,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:23.312470600Z",
@@ -2126,7 +2035,7 @@
      "data": {
       "text/plain": "(<Figure size 1600x400 with 1 Axes>,\n <Axes: ylabel='Number of airline passengers'>)"
      },
-     "execution_count": 58,
+     "execution_count": 55,
      "metadata": {},
      "output_type": "execute_result"
     },
@@ -2151,7 +2060,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 59,
+   "execution_count": 56,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:23.312470600Z",
@@ -2174,7 +2083,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 60,
+   "execution_count": 57,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:23.788198900Z",
@@ -2186,7 +2095,7 @@
      "data": {
       "text/plain": "(<Figure size 1600x400 with 1 Axes>,\n <Axes: ylabel='Number of airline passengers'>)"
      },
-     "execution_count": 60,
+     "execution_count": 57,
      "metadata": {},
      "output_type": "execute_result"
     },
@@ -2221,7 +2130,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 61,
+   "execution_count": 58,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:23.900898200Z",
@@ -2233,7 +2142,7 @@
      "data": {
       "text/plain": "1958-04    349.166352\n1958-05    346.925002\n1958-06    394.064152\n1958-07    435.855204\n1958-08    433.331082\n1958-09    384.856436\n1958-10    335.547256\n1958-11    293.184412\n1958-12    333.288639\n1959-01    338.608873\n1959-02    336.996809\n1959-03    388.137360\nFreq: M, dtype: float64"
      },
-     "execution_count": 61,
+     "execution_count": 58,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -2263,7 +2172,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 62,
+   "execution_count": 59,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:23.901895500Z",
@@ -2276,7 +2185,7 @@
       "text/plain": "AutoETS(auto=True, n_jobs=-1, sp=12)",
       "text/html": "<style>#sk-container-id-3 {color: black;background-color: white;}#sk-container-id-3 pre{padding: 0;}#sk-container-id-3 div.sk-toggleable {background-color: white;}#sk-container-id-3 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-3 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-3 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-3 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-3 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-3 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-3 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-3 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-3 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-3 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-3 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-3 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-3 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-3 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-3 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-3 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-3 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-3 div.sk-item {position: relative;z-index: 1;}#sk-container-id-3 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-3 div.sk-item::before, #sk-container-id-3 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-3 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-3 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-3 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-3 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-3 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-3 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-3 div.sk-label-container {text-align: center;}#sk-container-id-3 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-3 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-3\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>AutoETS(auto=True, n_jobs=-1, sp=12)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-3\" type=\"checkbox\" checked><label for=\"sk-estimator-id-3\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">AutoETS</label><div class=\"sk-toggleable__content\"><pre>AutoETS(auto=True, n_jobs=-1, sp=12)</pre></div></div></div></div></div>"
      },
-     "execution_count": 62,
+     "execution_count": 59,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -2306,7 +2215,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 63,
+   "execution_count": 60,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:23.901895500Z",
@@ -2330,7 +2239,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 64,
+   "execution_count": 61,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:23.902892Z",
@@ -2345,7 +2254,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 65,
+   "execution_count": 62,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:23.902892Z",
@@ -2387,7 +2296,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 66,
+   "execution_count": 63,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:23.948769Z",
@@ -2403,7 +2312,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 67,
+   "execution_count": 64,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:24.097372900Z",
@@ -2413,10 +2322,10 @@
    "outputs": [
     {
      "data": {
-      "text/plain": "   test_MeanAbsolutePercentageError  fit_time  pred_time  len_train_window  \\\n0                          0.083537  0.008972   0.006024                72   \n1                          0.047574  0.007817   0.006028                84   \n2                          0.059412  0.007609   0.005307                96   \n3                          0.040142  0.007615   0.005220               108   \n4                          0.101327  0.007433   0.005258               120   \n5                          0.051581  0.007838   0.005204               132   \n\n    cutoff  \n0  1954-12  \n1  1955-12  \n2  1956-12  \n3  1957-12  \n4  1958-12  \n5  1959-12  ",
-      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>test_MeanAbsolutePercentageError</th>\n      <th>fit_time</th>\n      <th>pred_time</th>\n      <th>len_train_window</th>\n      <th>cutoff</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0.083537</td>\n      <td>0.008972</td>\n      <td>0.006024</td>\n      <td>72</td>\n      <td>1954-12</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>0.047574</td>\n      <td>0.007817</td>\n      <td>0.006028</td>\n      <td>84</td>\n      <td>1955-12</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0.059412</td>\n      <td>0.007609</td>\n      <td>0.005307</td>\n      <td>96</td>\n      <td>1956-12</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>0.040142</td>\n      <td>0.007615</td>\n      <td>0.005220</td>\n      <td>108</td>\n      <td>1957-12</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>0.101327</td>\n      <td>0.007433</td>\n      <td>0.005258</td>\n      <td>120</td>\n      <td>1958-12</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>0.051581</td>\n      <td>0.007838</td>\n      <td>0.005204</td>\n      <td>132</td>\n      <td>1959-12</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
+      "text/plain": "   test_mean_absolute_percentage_error  fit_time  pred_time  len_train_window  \\\n0                             0.083537  0.014647   0.005874                72   \n1                             0.047574  0.008137   0.005779                84   \n2                             0.059412  0.008143   0.005170                96   \n3                             0.040142  0.007612   0.005146               108   \n4                             0.101327  0.007556   0.005259               120   \n5                             0.051581  0.007802   0.005211               132   \n\n    cutoff  \n0  1954-12  \n1  1955-12  \n2  1956-12  \n3  1957-12  \n4  1958-12  \n5  1959-12  ",
+      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>test_mean_absolute_percentage_error</th>\n      <th>fit_time</th>\n      <th>pred_time</th>\n      <th>len_train_window</th>\n      <th>cutoff</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0.083537</td>\n      <td>0.014647</td>\n      <td>0.005874</td>\n      <td>72</td>\n      <td>1954-12</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>0.047574</td>\n      <td>0.008137</td>\n      <td>0.005779</td>\n      <td>84</td>\n      <td>1955-12</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0.059412</td>\n      <td>0.008143</td>\n      <td>0.005170</td>\n      <td>96</td>\n      <td>1956-12</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>0.040142</td>\n      <td>0.007612</td>\n      <td>0.005146</td>\n      <td>108</td>\n      <td>1957-12</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>0.101327</td>\n      <td>0.007556</td>\n      <td>0.005259</td>\n      <td>120</td>\n      <td>1958-12</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>0.051581</td>\n      <td>0.007802</td>\n      <td>0.005211</td>\n      <td>132</td>\n      <td>1959-12</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
      },
-     "execution_count": 67,
+     "execution_count": 64,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -2435,7 +2344,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 68,
+   "execution_count": 65,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:25.285197Z",
@@ -2445,9 +2354,9 @@
    "outputs": [
     {
      "data": {
-      "text/plain": "<matplotlib.legend.Legend at 0x1a6d4e46c70>"
+      "text/plain": "<matplotlib.legend.Legend at 0x29e2d1f9550>"
      },
-     "execution_count": 68,
+     "execution_count": 65,
      "metadata": {},
      "output_type": "execute_result"
     },
@@ -2509,7 +2418,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 69,
+   "execution_count": 66,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:25.471698500Z",
@@ -2522,7 +2431,7 @@
       "text/plain": "                                         name  \\\n0                                        ARDL   \n1                                       ARIMA   \n2                                   AutoARIMA   \n3                                     AutoETS   \n4                      AutoEnsembleForecaster   \n5                                        BATS   \n6                           BaggingForecaster   \n7                    ColumnEnsembleForecaster   \n8                          ConformalIntervals   \n9                                     Croston   \n10          DirRecTabularRegressionForecaster   \n11       DirRecTimeSeriesRegressionForecaster   \n12          DirectTabularRegressionForecaster   \n13       DirectTimeSeriesRegressionForecaster   \n14                                 DontUpdate   \n15                              DynamicFactor   \n16                         EnsembleForecaster   \n17                       ExponentialSmoothing   \n18                            ForecastByLevel   \n19                                  ForecastX   \n20                    ForecastingGridSearchCV   \n21                        ForecastingPipeline   \n22              ForecastingRandomizedSearchCV   \n23                HierarchyEnsembleForecaster   \n24                             MockForecaster   \n25             MockUnivariateForecasterLogger   \n26     MultioutputTabularRegressionForecaster   \n27  MultioutputTimeSeriesRegressionForecaster   \n28                        MultiplexForecaster   \n29                            NaiveForecaster   \n30                              NaiveVariance   \n31                   OnlineEnsembleForecaster   \n32                                    Permute   \n33                  PolynomialTrendForecaster   \n34                                    Prophet   \n35                       ReconcilerForecaster   \n36       RecursiveTabularRegressionForecaster   \n37    RecursiveTimeSeriesRegressionForecaster   \n38                                    SARIMAX   \n39                              STLForecaster   \n40                          SquaringResiduals   \n41                         StackingForecaster   \n42                     StatsForecastAutoARIMA   \n43                                      TBATS   \n44                            ThetaForecaster   \n45                     ThetaModularForecaster   \n46                TransformedTargetForecaster   \n47                            TrendForecaster   \n48                       UnobservedComponents   \n49                                UpdateEvery   \n50                          UpdateRefitsEvery   \n51                                        VAR   \n52                                     VARMAX   \n53                                       VECM   \n\n                                            estimator  \n0                <class 'aeon.forecasting.ardl.ARDL'>  \n1              <class 'aeon.forecasting.arima.ARIMA'>  \n2          <class 'aeon.forecasting.arima.AutoARIMA'>  \n3              <class 'aeon.forecasting.ets.AutoETS'>  \n4   <class 'aeon.forecasting.compose._ensemble.Aut...  \n5                <class 'aeon.forecasting.bats.BATS'>  \n6   <class 'aeon.forecasting.compose._bagging.Bagg...  \n7   <class 'aeon.forecasting.compose._column_ensem...  \n8   <class 'aeon.forecasting.conformal.ConformalIn...  \n9          <class 'aeon.forecasting.croston.Croston'>  \n10  <class 'aeon.forecasting.compose._reduce.DirRe...  \n11  <class 'aeon.forecasting.compose._reduce.DirRe...  \n12  <class 'aeon.forecasting.compose._reduce.Direc...  \n13  <class 'aeon.forecasting.compose._reduce.Direc...  \n14  <class 'aeon.forecasting.stream._update.DontUp...  \n15  <class 'aeon.forecasting.dynamic_factor.Dynami...  \n16  <class 'aeon.forecasting.compose._ensemble.Ens...  \n17  <class 'aeon.forecasting.exp_smoothing.Exponen...  \n18  <class 'aeon.forecasting.compose._grouped.Fore...  \n19  <class 'aeon.forecasting.compose._pipeline.For...  \n20  <class 'aeon.forecasting.model_selection._tune...  \n21  <class 'aeon.forecasting.compose._pipeline.For...  \n22  <class 'aeon.forecasting.model_selection._tune...  \n23  <class 'aeon.forecasting.compose._hierarchy_en...  \n24  <class 'aeon.testing.mock_estimators._mock_for...  \n25  <class 'aeon.testing.mock_estimators._mock_for...  \n26  <class 'aeon.forecasting.compose._reduce.Multi...  \n27  <class 'aeon.forecasting.compose._reduce.Multi...  \n28  <class 'aeon.forecasting.compose._multiplexer....  \n29   <class 'aeon.forecasting.naive.NaiveForecaster'>  \n30     <class 'aeon.forecasting.naive.NaiveVariance'>  \n31  <class 'aeon.forecasting.online_learning._onli...  \n32  <class 'aeon.forecasting.compose._pipeline.Per...  \n33  <class 'aeon.forecasting.trend.PolynomialTrend...  \n34       <class 'aeon.forecasting.fbprophet.Prophet'>  \n35  <class 'aeon.forecasting.reconcile.ReconcilerF...  \n36  <class 'aeon.forecasting.compose._reduce.Recur...  \n37  <class 'aeon.forecasting.compose._reduce.Recur...  \n38         <class 'aeon.forecasting.sarimax.SARIMAX'>  \n39     <class 'aeon.forecasting.trend.STLForecaster'>  \n40  <class 'aeon.forecasting.squaring_residuals.Sq...  \n41  <class 'aeon.forecasting.compose._stack.Stacki...  \n42  <class 'aeon.forecasting.statsforecast.StatsFo...  \n43             <class 'aeon.forecasting.tbats.TBATS'>  \n44   <class 'aeon.forecasting.theta.ThetaForecaster'>  \n45  <class 'aeon.forecasting.theta.ThetaModularFor...  \n46  <class 'aeon.forecasting.compose._pipeline.Tra...  \n47   <class 'aeon.forecasting.trend.TrendForecaster'>  \n48  <class 'aeon.forecasting.structural.Unobserved...  \n49  <class 'aeon.forecasting.stream._update.Update...  \n50  <class 'aeon.forecasting.stream._update.Update...  \n51                 <class 'aeon.forecasting.var.VAR'>  \n52           <class 'aeon.forecasting.varmax.VARMAX'>  \n53               <class 'aeon.forecasting.vecm.VECM'>  ",
       "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>name</th>\n      <th>estimator</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>ARDL</td>\n      <td>&lt;class 'aeon.forecasting.ardl.ARDL'&gt;</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>ARIMA</td>\n      <td>&lt;class 'aeon.forecasting.arima.ARIMA'&gt;</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>AutoARIMA</td>\n      <td>&lt;class 'aeon.forecasting.arima.AutoARIMA'&gt;</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>AutoETS</td>\n      <td>&lt;class 'aeon.forecasting.ets.AutoETS'&gt;</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>AutoEnsembleForecaster</td>\n      <td>&lt;class 'aeon.forecasting.compose._ensemble.Aut...</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>BATS</td>\n      <td>&lt;class 'aeon.forecasting.bats.BATS'&gt;</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>BaggingForecaster</td>\n      <td>&lt;class 'aeon.forecasting.compose._bagging.Bagg...</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>ColumnEnsembleForecaster</td>\n      <td>&lt;class 'aeon.forecasting.compose._column_ensem...</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>ConformalIntervals</td>\n      <td>&lt;class 'aeon.forecasting.conformal.ConformalIn...</td>\n    </tr>\n    <tr>\n      <th>9</th>\n      <td>Croston</td>\n      <td>&lt;class 'aeon.forecasting.croston.Croston'&gt;</td>\n    </tr>\n    <tr>\n      <th>10</th>\n      <td>DirRecTabularRegressionForecaster</td>\n      <td>&lt;class 'aeon.forecasting.compose._reduce.DirRe...</td>\n    </tr>\n    <tr>\n      <th>11</th>\n      <td>DirRecTimeSeriesRegressionForecaster</td>\n      <td>&lt;class 'aeon.forecasting.compose._reduce.DirRe...</td>\n    </tr>\n    <tr>\n      <th>12</th>\n      <td>DirectTabularRegressionForecaster</td>\n      <td>&lt;class 'aeon.forecasting.compose._reduce.Direc...</td>\n    </tr>\n    <tr>\n      <th>13</th>\n      <td>DirectTimeSeriesRegressionForecaster</td>\n      <td>&lt;class 'aeon.forecasting.compose._reduce.Direc...</td>\n    </tr>\n    <tr>\n      <th>14</th>\n      <td>DontUpdate</td>\n      <td>&lt;class 'aeon.forecasting.stream._update.DontUp...</td>\n    </tr>\n    <tr>\n      <th>15</th>\n      <td>DynamicFactor</td>\n      <td>&lt;class 'aeon.forecasting.dynamic_factor.Dynami...</td>\n    </tr>\n    <tr>\n      <th>16</th>\n      <td>EnsembleForecaster</td>\n      <td>&lt;class 'aeon.forecasting.compose._ensemble.Ens...</td>\n    </tr>\n    <tr>\n      <th>17</th>\n      <td>ExponentialSmoothing</td>\n      <td>&lt;class 'aeon.forecasting.exp_smoothing.Exponen...</td>\n    </tr>\n    <tr>\n      <th>18</th>\n      <td>ForecastByLevel</td>\n      <td>&lt;class 'aeon.forecasting.compose._grouped.Fore...</td>\n    </tr>\n    <tr>\n      <th>19</th>\n      <td>ForecastX</td>\n      <td>&lt;class 'aeon.forecasting.compose._pipeline.For...</td>\n    </tr>\n    <tr>\n      <th>20</th>\n      <td>ForecastingGridSearchCV</td>\n      <td>&lt;class 'aeon.forecasting.model_selection._tune...</td>\n    </tr>\n    <tr>\n      <th>21</th>\n      <td>ForecastingPipeline</td>\n      <td>&lt;class 'aeon.forecasting.compose._pipeline.For...</td>\n    </tr>\n    <tr>\n      <th>22</th>\n      <td>ForecastingRandomizedSearchCV</td>\n      <td>&lt;class 'aeon.forecasting.model_selection._tune...</td>\n    </tr>\n    <tr>\n      <th>23</th>\n      <td>HierarchyEnsembleForecaster</td>\n      <td>&lt;class 'aeon.forecasting.compose._hierarchy_en...</td>\n    </tr>\n    <tr>\n      <th>24</th>\n      <td>MockForecaster</td>\n      <td>&lt;class 'aeon.testing.mock_estimators._mock_for...</td>\n    </tr>\n    <tr>\n      <th>25</th>\n      <td>MockUnivariateForecasterLogger</td>\n      <td>&lt;class 'aeon.testing.mock_estimators._mock_for...</td>\n    </tr>\n    <tr>\n      <th>26</th>\n      <td>MultioutputTabularRegressionForecaster</td>\n      <td>&lt;class 'aeon.forecasting.compose._reduce.Multi...</td>\n    </tr>\n    <tr>\n      <th>27</th>\n      <td>MultioutputTimeSeriesRegressionForecaster</td>\n      <td>&lt;class 'aeon.forecasting.compose._reduce.Multi...</td>\n    </tr>\n    <tr>\n      <th>28</th>\n      <td>MultiplexForecaster</td>\n      <td>&lt;class 'aeon.forecasting.compose._multiplexer....</td>\n    </tr>\n    <tr>\n      <th>29</th>\n      <td>NaiveForecaster</td>\n      <td>&lt;class 'aeon.forecasting.naive.NaiveForecaster'&gt;</td>\n    </tr>\n    <tr>\n      <th>30</th>\n      <td>NaiveVariance</td>\n      <td>&lt;class 'aeon.forecasting.naive.NaiveVariance'&gt;</td>\n    </tr>\n    <tr>\n      <th>31</th>\n      <td>OnlineEnsembleForecaster</td>\n      <td>&lt;class 'aeon.forecasting.online_learning._onli...</td>\n    </tr>\n    <tr>\n      <th>32</th>\n      <td>Permute</td>\n      <td>&lt;class 'aeon.forecasting.compose._pipeline.Per...</td>\n    </tr>\n    <tr>\n      <th>33</th>\n      <td>PolynomialTrendForecaster</td>\n      <td>&lt;class 'aeon.forecasting.trend.PolynomialTrend...</td>\n    </tr>\n    <tr>\n      <th>34</th>\n      <td>Prophet</td>\n      <td>&lt;class 'aeon.forecasting.fbprophet.Prophet'&gt;</td>\n    </tr>\n    <tr>\n      <th>35</th>\n      <td>ReconcilerForecaster</td>\n      <td>&lt;class 'aeon.forecasting.reconcile.ReconcilerF...</td>\n    </tr>\n    <tr>\n      <th>36</th>\n      <td>RecursiveTabularRegressionForecaster</td>\n      <td>&lt;class 'aeon.forecasting.compose._reduce.Recur...</td>\n    </tr>\n    <tr>\n      <th>37</th>\n      <td>RecursiveTimeSeriesRegressionForecaster</td>\n      <td>&lt;class 'aeon.forecasting.compose._reduce.Recur...</td>\n    </tr>\n    <tr>\n      <th>38</th>\n      <td>SARIMAX</td>\n      <td>&lt;class 'aeon.forecasting.sarimax.SARIMAX'&gt;</td>\n    </tr>\n    <tr>\n      <th>39</th>\n      <td>STLForecaster</td>\n      <td>&lt;class 'aeon.forecasting.trend.STLForecaster'&gt;</td>\n    </tr>\n    <tr>\n      <th>40</th>\n      <td>SquaringResiduals</td>\n      <td>&lt;class 'aeon.forecasting.squaring_residuals.Sq...</td>\n    </tr>\n    <tr>\n      <th>41</th>\n      <td>StackingForecaster</td>\n      <td>&lt;class 'aeon.forecasting.compose._stack.Stacki...</td>\n    </tr>\n    <tr>\n      <th>42</th>\n      <td>StatsForecastAutoARIMA</td>\n      <td>&lt;class 'aeon.forecasting.statsforecast.StatsFo...</td>\n    </tr>\n    <tr>\n      <th>43</th>\n      <td>TBATS</td>\n      <td>&lt;class 'aeon.forecasting.tbats.TBATS'&gt;</td>\n    </tr>\n    <tr>\n      <th>44</th>\n      <td>ThetaForecaster</td>\n      <td>&lt;class 'aeon.forecasting.theta.ThetaForecaster'&gt;</td>\n    </tr>\n    <tr>\n      <th>45</th>\n      <td>ThetaModularForecaster</td>\n      <td>&lt;class 'aeon.forecasting.theta.ThetaModularFor...</td>\n    </tr>\n    <tr>\n      <th>46</th>\n      <td>TransformedTargetForecaster</td>\n      <td>&lt;class 'aeon.forecasting.compose._pipeline.Tra...</td>\n    </tr>\n    <tr>\n      <th>47</th>\n      <td>TrendForecaster</td>\n      <td>&lt;class 'aeon.forecasting.trend.TrendForecaster'&gt;</td>\n    </tr>\n    <tr>\n      <th>48</th>\n      <td>UnobservedComponents</td>\n      <td>&lt;class 'aeon.forecasting.structural.Unobserved...</td>\n    </tr>\n    <tr>\n      <th>49</th>\n      <td>UpdateEvery</td>\n      <td>&lt;class 'aeon.forecasting.stream._update.Update...</td>\n    </tr>\n    <tr>\n      <th>50</th>\n      <td>UpdateRefitsEvery</td>\n      <td>&lt;class 'aeon.forecasting.stream._update.Update...</td>\n    </tr>\n    <tr>\n      <th>51</th>\n      <td>VAR</td>\n      <td>&lt;class 'aeon.forecasting.var.VAR'&gt;</td>\n    </tr>\n    <tr>\n      <th>52</th>\n      <td>VARMAX</td>\n      <td>&lt;class 'aeon.forecasting.varmax.VARMAX'&gt;</td>\n    </tr>\n    <tr>\n      <th>53</th>\n      <td>VECM</td>\n      <td>&lt;class 'aeon.forecasting.vecm.VECM'&gt;</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
      },
-     "execution_count": 69,
+     "execution_count": 66,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -2581,7 +2490,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 70,
+   "execution_count": 67,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:25.592375200Z",
@@ -2593,7 +2502,7 @@
      "data": {
       "text/plain": "{'y_input_type': 'univariate',\n 'ignores-exogeneous-X': False,\n 'capability:pred_int': True,\n 'capability:missing_values': True,\n 'y_inner_type': 'pd.Series',\n 'X_inner_type': 'pd.DataFrame',\n 'requires-fh-in-fit': False,\n 'X-y-must-have-same-index': True,\n 'enforce_index_type': None,\n 'fit_is_empty': False,\n 'python_version': None,\n 'python_dependencies': 'pmdarima'}"
      },
-     "execution_count": 70,
+     "execution_count": 67,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -2631,7 +2540,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 71,
+   "execution_count": 68,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:25.593373300Z",
@@ -2644,7 +2553,7 @@
       "text/plain": "                                         name  \\\n0                                        ARDL   \n1                                       ARIMA   \n2                                   AutoARIMA   \n3                                     AutoETS   \n4                      AutoEnsembleForecaster   \n5                                        BATS   \n6                           BaggingForecaster   \n7                    ColumnEnsembleForecaster   \n8                          ConformalIntervals   \n9                                     Croston   \n10          DirRecTabularRegressionForecaster   \n11       DirRecTimeSeriesRegressionForecaster   \n12          DirectTabularRegressionForecaster   \n13       DirectTimeSeriesRegressionForecaster   \n14                                 DontUpdate   \n15                              DynamicFactor   \n16                         EnsembleForecaster   \n17                       ExponentialSmoothing   \n18                            ForecastByLevel   \n19                                  ForecastX   \n20                    ForecastingGridSearchCV   \n21                        ForecastingPipeline   \n22              ForecastingRandomizedSearchCV   \n23                HierarchyEnsembleForecaster   \n24                             MockForecaster   \n25             MockUnivariateForecasterLogger   \n26     MultioutputTabularRegressionForecaster   \n27  MultioutputTimeSeriesRegressionForecaster   \n28                        MultiplexForecaster   \n29                            NaiveForecaster   \n30                              NaiveVariance   \n31                   OnlineEnsembleForecaster   \n32                                    Permute   \n33                  PolynomialTrendForecaster   \n34                                    Prophet   \n35                       ReconcilerForecaster   \n36       RecursiveTabularRegressionForecaster   \n37    RecursiveTimeSeriesRegressionForecaster   \n38                                    SARIMAX   \n39                              STLForecaster   \n40                          SquaringResiduals   \n41                         StackingForecaster   \n42                     StatsForecastAutoARIMA   \n43                                      TBATS   \n44                            ThetaForecaster   \n45                     ThetaModularForecaster   \n46                TransformedTargetForecaster   \n47                            TrendForecaster   \n48                       UnobservedComponents   \n49                                UpdateEvery   \n50                          UpdateRefitsEvery   \n51                                        VAR   \n52                                     VARMAX   \n53                                       VECM   \n\n                                            estimator  y_input_type  \\\n0                <class 'aeon.forecasting.ardl.ARDL'>    univariate   \n1              <class 'aeon.forecasting.arima.ARIMA'>    univariate   \n2          <class 'aeon.forecasting.arima.AutoARIMA'>    univariate   \n3              <class 'aeon.forecasting.ets.AutoETS'>    univariate   \n4   <class 'aeon.forecasting.compose._ensemble.Aut...    univariate   \n5                <class 'aeon.forecasting.bats.BATS'>    univariate   \n6   <class 'aeon.forecasting.compose._bagging.Bagg...    univariate   \n7   <class 'aeon.forecasting.compose._column_ensem...          both   \n8   <class 'aeon.forecasting.conformal.ConformalIn...    univariate   \n9          <class 'aeon.forecasting.croston.Croston'>    univariate   \n10  <class 'aeon.forecasting.compose._reduce.DirRe...    univariate   \n11  <class 'aeon.forecasting.compose._reduce.DirRe...    univariate   \n12  <class 'aeon.forecasting.compose._reduce.Direc...    univariate   \n13  <class 'aeon.forecasting.compose._reduce.Direc...    univariate   \n14  <class 'aeon.forecasting.stream._update.DontUp...    univariate   \n15  <class 'aeon.forecasting.dynamic_factor.Dynami...  multivariate   \n16  <class 'aeon.forecasting.compose._ensemble.Ens...          both   \n17  <class 'aeon.forecasting.exp_smoothing.Exponen...    univariate   \n18  <class 'aeon.forecasting.compose._grouped.Fore...          both   \n19  <class 'aeon.forecasting.compose._pipeline.For...    univariate   \n20  <class 'aeon.forecasting.model_selection._tune...          both   \n21  <class 'aeon.forecasting.compose._pipeline.For...          both   \n22  <class 'aeon.forecasting.model_selection._tune...          both   \n23  <class 'aeon.forecasting.compose._hierarchy_en...          both   \n24  <class 'aeon.testing.mock_estimators._mock_for...          both   \n25  <class 'aeon.testing.mock_estimators._mock_for...    univariate   \n26  <class 'aeon.forecasting.compose._reduce.Multi...    univariate   \n27  <class 'aeon.forecasting.compose._reduce.Multi...    univariate   \n28  <class 'aeon.forecasting.compose._multiplexer....          both   \n29   <class 'aeon.forecasting.naive.NaiveForecaster'>    univariate   \n30     <class 'aeon.forecasting.naive.NaiveVariance'>    univariate   \n31  <class 'aeon.forecasting.online_learning._onli...    univariate   \n32  <class 'aeon.forecasting.compose._pipeline.Per...          both   \n33  <class 'aeon.forecasting.trend.PolynomialTrend...    univariate   \n34       <class 'aeon.forecasting.fbprophet.Prophet'>    univariate   \n35  <class 'aeon.forecasting.reconcile.ReconcilerF...    univariate   \n36  <class 'aeon.forecasting.compose._reduce.Recur...    univariate   \n37  <class 'aeon.forecasting.compose._reduce.Recur...    univariate   \n38         <class 'aeon.forecasting.sarimax.SARIMAX'>    univariate   \n39     <class 'aeon.forecasting.trend.STLForecaster'>    univariate   \n40  <class 'aeon.forecasting.squaring_residuals.Sq...    univariate   \n41  <class 'aeon.forecasting.compose._stack.Stacki...    univariate   \n42  <class 'aeon.forecasting.statsforecast.StatsFo...    univariate   \n43             <class 'aeon.forecasting.tbats.TBATS'>    univariate   \n44   <class 'aeon.forecasting.theta.ThetaForecaster'>    univariate   \n45  <class 'aeon.forecasting.theta.ThetaModularFor...    univariate   \n46  <class 'aeon.forecasting.compose._pipeline.Tra...          both   \n47   <class 'aeon.forecasting.trend.TrendForecaster'>    univariate   \n48  <class 'aeon.forecasting.structural.Unobserved...    univariate   \n49  <class 'aeon.forecasting.stream._update.Update...    univariate   \n50  <class 'aeon.forecasting.stream._update.Update...    univariate   \n51                 <class 'aeon.forecasting.var.VAR'>  multivariate   \n52           <class 'aeon.forecasting.varmax.VARMAX'>  multivariate   \n53               <class 'aeon.forecasting.vecm.VECM'>  multivariate   \n\n    requires-fh-in-fit  \n0                False  \n1                False  \n2                False  \n3                False  \n4                False  \n5                False  \n6                False  \n7                False  \n8                False  \n9                False  \n10                True  \n11                True  \n12                True  \n13                True  \n14               False  \n15               False  \n16               False  \n17               False  \n18               False  \n19                True  \n20               False  \n21               False  \n22               False  \n23               False  \n24               False  \n25               False  \n26                True  \n27                True  \n28               False  \n29               False  \n30               False  \n31               False  \n32               False  \n33               False  \n34               False  \n35               False  \n36               False  \n37               False  \n38               False  \n39               False  \n40                True  \n41                True  \n42               False  \n43               False  \n44               False  \n45               False  \n46               False  \n47               False  \n48               False  \n49               False  \n50               False  \n51               False  \n52               False  \n53               False  ",
       "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>name</th>\n      <th>estimator</th>\n      <th>y_input_type</th>\n      <th>requires-fh-in-fit</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>ARDL</td>\n      <td>&lt;class 'aeon.forecasting.ardl.ARDL'&gt;</td>\n      <td>univariate</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>ARIMA</td>\n      <td>&lt;class 'aeon.forecasting.arima.ARIMA'&gt;</td>\n      <td>univariate</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>AutoARIMA</td>\n      <td>&lt;class 'aeon.forecasting.arima.AutoARIMA'&gt;</td>\n      <td>univariate</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>AutoETS</td>\n      <td>&lt;class 'aeon.forecasting.ets.AutoETS'&gt;</td>\n      <td>univariate</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>AutoEnsembleForecaster</td>\n      <td>&lt;class 'aeon.forecasting.compose._ensemble.Aut...</td>\n      <td>univariate</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>BATS</td>\n      <td>&lt;class 'aeon.forecasting.bats.BATS'&gt;</td>\n      <td>univariate</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>BaggingForecaster</td>\n      <td>&lt;class 'aeon.forecasting.compose._bagging.Bagg...</td>\n      <td>univariate</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>ColumnEnsembleForecaster</td>\n      <td>&lt;class 'aeon.forecasting.compose._column_ensem...</td>\n      <td>both</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>ConformalIntervals</td>\n      <td>&lt;class 'aeon.forecasting.conformal.ConformalIn...</td>\n      <td>univariate</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>9</th>\n      <td>Croston</td>\n      <td>&lt;class 'aeon.forecasting.croston.Croston'&gt;</td>\n      <td>univariate</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>10</th>\n      <td>DirRecTabularRegressionForecaster</td>\n      <td>&lt;class 'aeon.forecasting.compose._reduce.DirRe...</td>\n      <td>univariate</td>\n      <td>True</td>\n    </tr>\n    <tr>\n      <th>11</th>\n      <td>DirRecTimeSeriesRegressionForecaster</td>\n      <td>&lt;class 'aeon.forecasting.compose._reduce.DirRe...</td>\n      <td>univariate</td>\n      <td>True</td>\n    </tr>\n    <tr>\n      <th>12</th>\n      <td>DirectTabularRegressionForecaster</td>\n      <td>&lt;class 'aeon.forecasting.compose._reduce.Direc...</td>\n      <td>univariate</td>\n      <td>True</td>\n    </tr>\n    <tr>\n      <th>13</th>\n      <td>DirectTimeSeriesRegressionForecaster</td>\n      <td>&lt;class 'aeon.forecasting.compose._reduce.Direc...</td>\n      <td>univariate</td>\n      <td>True</td>\n    </tr>\n    <tr>\n      <th>14</th>\n      <td>DontUpdate</td>\n      <td>&lt;class 'aeon.forecasting.stream._update.DontUp...</td>\n      <td>univariate</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>15</th>\n      <td>DynamicFactor</td>\n      <td>&lt;class 'aeon.forecasting.dynamic_factor.Dynami...</td>\n      <td>multivariate</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>16</th>\n      <td>EnsembleForecaster</td>\n      <td>&lt;class 'aeon.forecasting.compose._ensemble.Ens...</td>\n      <td>both</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>17</th>\n      <td>ExponentialSmoothing</td>\n      <td>&lt;class 'aeon.forecasting.exp_smoothing.Exponen...</td>\n      <td>univariate</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>18</th>\n      <td>ForecastByLevel</td>\n      <td>&lt;class 'aeon.forecasting.compose._grouped.Fore...</td>\n      <td>both</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>19</th>\n      <td>ForecastX</td>\n      <td>&lt;class 'aeon.forecasting.compose._pipeline.For...</td>\n      <td>univariate</td>\n      <td>True</td>\n    </tr>\n    <tr>\n      <th>20</th>\n      <td>ForecastingGridSearchCV</td>\n      <td>&lt;class 'aeon.forecasting.model_selection._tune...</td>\n      <td>both</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>21</th>\n      <td>ForecastingPipeline</td>\n      <td>&lt;class 'aeon.forecasting.compose._pipeline.For...</td>\n      <td>both</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>22</th>\n      <td>ForecastingRandomizedSearchCV</td>\n      <td>&lt;class 'aeon.forecasting.model_selection._tune...</td>\n      <td>both</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>23</th>\n      <td>HierarchyEnsembleForecaster</td>\n      <td>&lt;class 'aeon.forecasting.compose._hierarchy_en...</td>\n      <td>both</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>24</th>\n      <td>MockForecaster</td>\n      <td>&lt;class 'aeon.testing.mock_estimators._mock_for...</td>\n      <td>both</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>25</th>\n      <td>MockUnivariateForecasterLogger</td>\n      <td>&lt;class 'aeon.testing.mock_estimators._mock_for...</td>\n      <td>univariate</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>26</th>\n      <td>MultioutputTabularRegressionForecaster</td>\n      <td>&lt;class 'aeon.forecasting.compose._reduce.Multi...</td>\n      <td>univariate</td>\n      <td>True</td>\n    </tr>\n    <tr>\n      <th>27</th>\n      <td>MultioutputTimeSeriesRegressionForecaster</td>\n      <td>&lt;class 'aeon.forecasting.compose._reduce.Multi...</td>\n      <td>univariate</td>\n      <td>True</td>\n    </tr>\n    <tr>\n      <th>28</th>\n      <td>MultiplexForecaster</td>\n      <td>&lt;class 'aeon.forecasting.compose._multiplexer....</td>\n      <td>both</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>29</th>\n      <td>NaiveForecaster</td>\n      <td>&lt;class 'aeon.forecasting.naive.NaiveForecaster'&gt;</td>\n      <td>univariate</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>30</th>\n      <td>NaiveVariance</td>\n      <td>&lt;class 'aeon.forecasting.naive.NaiveVariance'&gt;</td>\n      <td>univariate</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>31</th>\n      <td>OnlineEnsembleForecaster</td>\n      <td>&lt;class 'aeon.forecasting.online_learning._onli...</td>\n      <td>univariate</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>32</th>\n      <td>Permute</td>\n      <td>&lt;class 'aeon.forecasting.compose._pipeline.Per...</td>\n      <td>both</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>33</th>\n      <td>PolynomialTrendForecaster</td>\n      <td>&lt;class 'aeon.forecasting.trend.PolynomialTrend...</td>\n      <td>univariate</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>34</th>\n      <td>Prophet</td>\n      <td>&lt;class 'aeon.forecasting.fbprophet.Prophet'&gt;</td>\n      <td>univariate</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>35</th>\n      <td>ReconcilerForecaster</td>\n      <td>&lt;class 'aeon.forecasting.reconcile.ReconcilerF...</td>\n      <td>univariate</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>36</th>\n      <td>RecursiveTabularRegressionForecaster</td>\n      <td>&lt;class 'aeon.forecasting.compose._reduce.Recur...</td>\n      <td>univariate</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>37</th>\n      <td>RecursiveTimeSeriesRegressionForecaster</td>\n      <td>&lt;class 'aeon.forecasting.compose._reduce.Recur...</td>\n      <td>univariate</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>38</th>\n      <td>SARIMAX</td>\n      <td>&lt;class 'aeon.forecasting.sarimax.SARIMAX'&gt;</td>\n      <td>univariate</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>39</th>\n      <td>STLForecaster</td>\n      <td>&lt;class 'aeon.forecasting.trend.STLForecaster'&gt;</td>\n      <td>univariate</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>40</th>\n      <td>SquaringResiduals</td>\n      <td>&lt;class 'aeon.forecasting.squaring_residuals.Sq...</td>\n      <td>univariate</td>\n      <td>True</td>\n    </tr>\n    <tr>\n      <th>41</th>\n      <td>StackingForecaster</td>\n      <td>&lt;class 'aeon.forecasting.compose._stack.Stacki...</td>\n      <td>univariate</td>\n      <td>True</td>\n    </tr>\n    <tr>\n      <th>42</th>\n      <td>StatsForecastAutoARIMA</td>\n      <td>&lt;class 'aeon.forecasting.statsforecast.StatsFo...</td>\n      <td>univariate</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>43</th>\n      <td>TBATS</td>\n      <td>&lt;class 'aeon.forecasting.tbats.TBATS'&gt;</td>\n      <td>univariate</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>44</th>\n      <td>ThetaForecaster</td>\n      <td>&lt;class 'aeon.forecasting.theta.ThetaForecaster'&gt;</td>\n      <td>univariate</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>45</th>\n      <td>ThetaModularForecaster</td>\n      <td>&lt;class 'aeon.forecasting.theta.ThetaModularFor...</td>\n      <td>univariate</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>46</th>\n      <td>TransformedTargetForecaster</td>\n      <td>&lt;class 'aeon.forecasting.compose._pipeline.Tra...</td>\n      <td>both</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>47</th>\n      <td>TrendForecaster</td>\n      <td>&lt;class 'aeon.forecasting.trend.TrendForecaster'&gt;</td>\n      <td>univariate</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>48</th>\n      <td>UnobservedComponents</td>\n      <td>&lt;class 'aeon.forecasting.structural.Unobserved...</td>\n      <td>univariate</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>49</th>\n      <td>UpdateEvery</td>\n      <td>&lt;class 'aeon.forecasting.stream._update.Update...</td>\n      <td>univariate</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>50</th>\n      <td>UpdateRefitsEvery</td>\n      <td>&lt;class 'aeon.forecasting.stream._update.Update...</td>\n      <td>univariate</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>51</th>\n      <td>VAR</td>\n      <td>&lt;class 'aeon.forecasting.var.VAR'&gt;</td>\n      <td>multivariate</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>52</th>\n      <td>VARMAX</td>\n      <td>&lt;class 'aeon.forecasting.varmax.VARMAX'&gt;</td>\n      <td>multivariate</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>53</th>\n      <td>VECM</td>\n      <td>&lt;class 'aeon.forecasting.vecm.VECM'&gt;</td>\n      <td>multivariate</td>\n      <td>False</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
      },
-     "execution_count": 71,
+     "execution_count": 68,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -2666,7 +2575,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 72,
+   "execution_count": 69,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:25.594370700Z",
@@ -2679,7 +2588,7 @@
       "text/plain": "                             name  \\\n0        ColumnEnsembleForecaster   \n1                   DynamicFactor   \n2              EnsembleForecaster   \n3                 ForecastByLevel   \n4         ForecastingGridSearchCV   \n5             ForecastingPipeline   \n6   ForecastingRandomizedSearchCV   \n7     HierarchyEnsembleForecaster   \n8                  MockForecaster   \n9             MultiplexForecaster   \n10                        Permute   \n11    TransformedTargetForecaster   \n12                            VAR   \n13                         VARMAX   \n14                           VECM   \n\n                                            estimator  \n0   <class 'aeon.forecasting.compose._column_ensem...  \n1   <class 'aeon.forecasting.dynamic_factor.Dynami...  \n2   <class 'aeon.forecasting.compose._ensemble.Ens...  \n3   <class 'aeon.forecasting.compose._grouped.Fore...  \n4   <class 'aeon.forecasting.model_selection._tune...  \n5   <class 'aeon.forecasting.compose._pipeline.For...  \n6   <class 'aeon.forecasting.model_selection._tune...  \n7   <class 'aeon.forecasting.compose._hierarchy_en...  \n8   <class 'aeon.testing.mock_estimators._mock_for...  \n9   <class 'aeon.forecasting.compose._multiplexer....  \n10  <class 'aeon.forecasting.compose._pipeline.Per...  \n11  <class 'aeon.forecasting.compose._pipeline.Tra...  \n12                 <class 'aeon.forecasting.var.VAR'>  \n13           <class 'aeon.forecasting.varmax.VARMAX'>  \n14               <class 'aeon.forecasting.vecm.VECM'>  ",
       "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>name</th>\n      <th>estimator</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>ColumnEnsembleForecaster</td>\n      <td>&lt;class 'aeon.forecasting.compose._column_ensem...</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>DynamicFactor</td>\n      <td>&lt;class 'aeon.forecasting.dynamic_factor.Dynami...</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>EnsembleForecaster</td>\n      <td>&lt;class 'aeon.forecasting.compose._ensemble.Ens...</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>ForecastByLevel</td>\n      <td>&lt;class 'aeon.forecasting.compose._grouped.Fore...</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>ForecastingGridSearchCV</td>\n      <td>&lt;class 'aeon.forecasting.model_selection._tune...</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>ForecastingPipeline</td>\n      <td>&lt;class 'aeon.forecasting.compose._pipeline.For...</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>ForecastingRandomizedSearchCV</td>\n      <td>&lt;class 'aeon.forecasting.model_selection._tune...</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>HierarchyEnsembleForecaster</td>\n      <td>&lt;class 'aeon.forecasting.compose._hierarchy_en...</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>MockForecaster</td>\n      <td>&lt;class 'aeon.testing.mock_estimators._mock_for...</td>\n    </tr>\n    <tr>\n      <th>9</th>\n      <td>MultiplexForecaster</td>\n      <td>&lt;class 'aeon.forecasting.compose._multiplexer....</td>\n    </tr>\n    <tr>\n      <th>10</th>\n      <td>Permute</td>\n      <td>&lt;class 'aeon.forecasting.compose._pipeline.Per...</td>\n    </tr>\n    <tr>\n      <th>11</th>\n      <td>TransformedTargetForecaster</td>\n      <td>&lt;class 'aeon.forecasting.compose._pipeline.Tra...</td>\n    </tr>\n    <tr>\n      <th>12</th>\n      <td>VAR</td>\n      <td>&lt;class 'aeon.forecasting.var.VAR'&gt;</td>\n    </tr>\n    <tr>\n      <th>13</th>\n      <td>VARMAX</td>\n      <td>&lt;class 'aeon.forecasting.varmax.VARMAX'&gt;</td>\n    </tr>\n    <tr>\n      <th>14</th>\n      <td>VECM</td>\n      <td>&lt;class 'aeon.forecasting.vecm.VECM'&gt;</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
      },
-     "execution_count": 72,
+     "execution_count": 69,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -2718,7 +2627,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 73,
+   "execution_count": 70,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:25.595368Z",
@@ -2731,7 +2640,7 @@
       "text/plain": "                          0                                                  3\n0  X-y-must-have-same-index  do X/y in fit/update and X/fh in predict have ...\n1              X_inner_type  which data structure is the internal _fit/_pre...\n2       capability:pred_int  does the forecaster implement predict_interval...\n3       capability:pred_var    does the forecaster implement predict_variance?\n4        enforce_index_type  passed to input checks, input conversion index...\n5      ignores-exogeneous-X        does forecaster ignore exogeneous data (X)?\n6             remember_data  whether estimator remembers all data seen as s...\n7        requires-fh-in-fit  does forecaster require fh passed already in f...\n8              y_inner_type  which data structure is the internal _fit/_pre...\n9              y_input_type  which series type does the forecaster support?...",
       "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>3</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>X-y-must-have-same-index</td>\n      <td>do X/y in fit/update and X/fh in predict have ...</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>X_inner_type</td>\n      <td>which data structure is the internal _fit/_pre...</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>capability:pred_int</td>\n      <td>does the forecaster implement predict_interval...</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>capability:pred_var</td>\n      <td>does the forecaster implement predict_variance?</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>enforce_index_type</td>\n      <td>passed to input checks, input conversion index...</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>ignores-exogeneous-X</td>\n      <td>does forecaster ignore exogeneous data (X)?</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>remember_data</td>\n      <td>whether estimator remembers all data seen as s...</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>requires-fh-in-fit</td>\n      <td>does forecaster require fh passed already in f...</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>y_inner_type</td>\n      <td>which data structure is the internal _fit/_pre...</td>\n    </tr>\n    <tr>\n      <th>9</th>\n      <td>y_input_type</td>\n      <td>which series type does the forecaster support?...</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
      },
-     "execution_count": 73,
+     "execution_count": 70,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -2771,7 +2680,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 74,
+   "execution_count": 71,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:25.595368Z",
@@ -2806,7 +2715,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 75,
+   "execution_count": 72,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:25.595368Z",
@@ -2820,7 +2729,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 76,
+   "execution_count": 73,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:26.242673200Z",
@@ -2832,7 +2741,7 @@
      "data": {
       "text/plain": "0.0511419028267833"
      },
-     "execution_count": 76,
+     "execution_count": 73,
      "metadata": {},
      "output_type": "execute_result"
     },
@@ -2864,7 +2773,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 77,
+   "execution_count": 74,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:26.242673200Z",
@@ -2878,7 +2787,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 78,
+   "execution_count": 75,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:26.984654200Z",
@@ -2890,7 +2799,7 @@
      "data": {
       "text/plain": "0.06186354612548242"
      },
-     "execution_count": 78,
+     "execution_count": 75,
      "metadata": {},
      "output_type": "execute_result"
     },
@@ -2926,7 +2835,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 79,
+   "execution_count": 76,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:26.985650700Z",
@@ -2940,7 +2849,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 80,
+   "execution_count": 77,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:26.987647200Z",
@@ -2952,7 +2861,7 @@
      "data": {
       "text/plain": "0.04356744949188808"
      },
-     "execution_count": 80,
+     "execution_count": 77,
      "metadata": {},
      "output_type": "execute_result"
     },
@@ -2985,7 +2894,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 81,
+   "execution_count": 78,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:26.988642900Z",
@@ -2999,7 +2908,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 82,
+   "execution_count": 79,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:28.794814600Z",
@@ -3011,7 +2920,7 @@
      "data": {
       "text/plain": "0.04148971434138202"
      },
-     "execution_count": 82,
+     "execution_count": 79,
      "metadata": {},
      "output_type": "execute_result"
     },
@@ -3035,7 +2944,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 83,
+   "execution_count": 80,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:30.969030500Z",
@@ -3047,7 +2956,7 @@
      "data": {
       "text/plain": "0.04093675930591672"
      },
-     "execution_count": 83,
+     "execution_count": 80,
      "metadata": {},
      "output_type": "execute_result"
     },
@@ -3071,7 +2980,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 84,
+   "execution_count": 81,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:30.975017600Z",
@@ -3083,7 +2992,7 @@
      "data": {
       "text/plain": "{'ar.L1': -0.2411177593160947,\n 'sigma2': 92.74985957133522,\n 'order': (1, 1, 0),\n 'seasonal_order': (0, 1, 0, 12),\n 'aic': 704.0011679025909,\n 'aicc': 704.1316026851996,\n 'bic': 709.108921685792,\n 'hqic': 706.0650836395923}"
      },
-     "execution_count": 84,
+     "execution_count": 81,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -3105,7 +3014,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 85,
+   "execution_count": 82,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:30.975017600Z",
@@ -3119,7 +3028,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 86,
+   "execution_count": 83,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:39.074334300Z",
@@ -3131,7 +3040,7 @@
      "data": {
       "text/plain": "0.08185559025534275"
      },
-     "execution_count": 86,
+     "execution_count": 83,
      "metadata": {},
      "output_type": "execute_result"
     },
@@ -3155,7 +3064,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 87,
+   "execution_count": 84,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:39.092286900Z",
@@ -3169,7 +3078,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 88,
+   "execution_count": 85,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:48.172014Z",
@@ -3181,7 +3090,7 @@
      "data": {
       "text/plain": "0.08024090844021753"
      },
-     "execution_count": 88,
+     "execution_count": 85,
      "metadata": {},
      "output_type": "execute_result"
     },
@@ -3214,7 +3123,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 89,
+   "execution_count": 86,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:48.177000800Z",
@@ -3235,7 +3144,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 90,
+   "execution_count": 87,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:48.207917600Z",
@@ -3252,7 +3161,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 91,
+   "execution_count": 88,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:48.924005800Z",
@@ -3264,15 +3173,15 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "20:17:50 - cmdstanpy - INFO - Chain [1] start processing\n",
-      "20:17:50 - cmdstanpy - INFO - Chain [1] done processing\n"
+      "16:52:01 - cmdstanpy - INFO - Chain [1] start processing\n",
+      "16:52:01 - cmdstanpy - INFO - Chain [1] done processing\n"
      ]
     },
     {
      "data": {
       "text/plain": "0.0714009805178252"
      },
-     "execution_count": 91,
+     "execution_count": 88,
      "metadata": {},
      "output_type": "execute_result"
     },
@@ -3314,7 +3223,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 92,
+   "execution_count": 89,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:48.927992700Z",
@@ -3328,7 +3237,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 93,
+   "execution_count": 90,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:49.629118900Z",
@@ -3340,7 +3249,7 @@
      "data": {
       "text/plain": "0.049737049410244434"
      },
-     "execution_count": 93,
+     "execution_count": 90,
      "metadata": {},
      "output_type": "execute_result"
     },
@@ -3385,7 +3294,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 94,
+   "execution_count": 91,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:49.694942500Z",
@@ -3433,7 +3342,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 95,
+   "execution_count": 139,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:49.706910400Z",
@@ -3449,7 +3358,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 96,
+   "execution_count": 140,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:49.863492800Z",
@@ -3464,7 +3373,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 97,
+   "execution_count": 141,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:50.134767900Z",
@@ -3476,7 +3385,7 @@
      "data": {
       "text/plain": "0.12887507224382988"
      },
-     "execution_count": 97,
+     "execution_count": 141,
      "metadata": {},
      "output_type": "execute_result"
     },
@@ -3515,7 +3424,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 98,
+   "execution_count": 95,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:50.135764300Z",
@@ -3527,7 +3436,7 @@
      "data": {
       "text/plain": "{'estimator__algorithm': 'auto',\n 'estimator__leaf_size': 30,\n 'estimator__metric': 'minkowski',\n 'estimator__metric_params': None,\n 'estimator__n_jobs': None,\n 'estimator__n_neighbors': 1,\n 'estimator__p': 2,\n 'estimator__weights': 'uniform',\n 'estimator': KNeighborsRegressor(n_neighbors=1),\n 'pooling': 'local',\n 'transformers': None,\n 'window_length': 15}"
      },
-     "execution_count": 98,
+     "execution_count": 95,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -3557,7 +3466,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 99,
+   "execution_count": 96,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:50.135764300Z",
@@ -3573,7 +3482,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 100,
+   "execution_count": 97,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:50.460895Z",
@@ -3585,7 +3494,7 @@
      "data": {
       "text/plain": "0.1396997352629755"
      },
-     "execution_count": 100,
+     "execution_count": 97,
      "metadata": {},
      "output_type": "execute_result"
     },
@@ -3630,7 +3539,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 101,
+   "execution_count": 98,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:50.461893300Z",
@@ -3643,7 +3552,7 @@
       "text/plain": "TransformedTargetForecaster(steps=[Deseasonalizer(model='multiplicative',\n                                                  sp=12),\n                                   ARIMA()])",
       "text/html": "<style>#sk-container-id-4 {color: black;background-color: white;}#sk-container-id-4 pre{padding: 0;}#sk-container-id-4 div.sk-toggleable {background-color: white;}#sk-container-id-4 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-4 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-4 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-4 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-4 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-4 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-4 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-4 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-4 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-4 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-4 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-4 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-4 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-4 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-4 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-4 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-4 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-4 div.sk-item {position: relative;z-index: 1;}#sk-container-id-4 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-4 div.sk-item::before, #sk-container-id-4 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-4 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-4 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-4 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-4 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-4 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-4 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-4 div.sk-label-container {text-align: center;}#sk-container-id-4 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-4 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-4\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>TransformedTargetForecaster(steps=[Deseasonalizer(model=&#x27;multiplicative&#x27;,\n                                                  sp=12),\n                                   ARIMA()])</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-4\" type=\"checkbox\" checked><label for=\"sk-estimator-id-4\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">TransformedTargetForecaster</label><div class=\"sk-toggleable__content\"><pre>TransformedTargetForecaster(steps=[Deseasonalizer(model=&#x27;multiplicative&#x27;,\n                                                  sp=12),\n                                   ARIMA()])</pre></div></div></div></div></div>"
      },
-     "execution_count": 101,
+     "execution_count": 98,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -3664,7 +3573,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 102,
+   "execution_count": 99,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:50.464883900Z",
@@ -3676,7 +3585,7 @@
      "data": {
       "text/plain": "{'steps': [Deseasonalizer(model='multiplicative', sp=12),\n  Deseasonalizer(model='multiplicative', sp=3),\n  ARIMA()],\n 'Deseasonalizer_1': Deseasonalizer(model='multiplicative', sp=12),\n 'Deseasonalizer_2': Deseasonalizer(model='multiplicative', sp=3),\n 'ARIMA': ARIMA(),\n 'Deseasonalizer_1__model': 'multiplicative',\n 'Deseasonalizer_1__sp': 12,\n 'Deseasonalizer_2__model': 'multiplicative',\n 'Deseasonalizer_2__sp': 3,\n 'ARIMA__concentrate_scale': False,\n 'ARIMA__enforce_invertibility': True,\n 'ARIMA__enforce_stationarity': True,\n 'ARIMA__hamilton_representation': False,\n 'ARIMA__maxiter': 50,\n 'ARIMA__measurement_error': False,\n 'ARIMA__method': 'lbfgs',\n 'ARIMA__mle_regression': True,\n 'ARIMA__order': (1, 0, 0),\n 'ARIMA__out_of_sample_size': 0,\n 'ARIMA__scoring': 'mse',\n 'ARIMA__scoring_args': None,\n 'ARIMA__seasonal_order': (0, 0, 0, 0),\n 'ARIMA__simple_differencing': False,\n 'ARIMA__start_params': None,\n 'ARIMA__suppress_warnings': False,\n 'ARIMA__time_varying_regression': False,\n 'ARIMA__trend': None,\n 'ARIMA__with_intercept': True}"
      },
-     "execution_count": 102,
+     "execution_count": 99,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -3702,7 +3611,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 103,
+   "execution_count": 100,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:50.768074200Z",
@@ -3714,7 +3623,7 @@
      "data": {
       "text/plain": "1958-01    334.0\n1958-02    338.0\n1958-03    317.0\nFreq: M, dtype: float64"
      },
-     "execution_count": 103,
+     "execution_count": 100,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -3735,7 +3644,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 104,
+   "execution_count": 101,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:50.828910700Z",
@@ -3747,7 +3656,7 @@
      "data": {
       "text/plain": "Period\n1958-01    339.0\n1958-02    334.0\n1958-03    381.0\nFreq: M, dtype: float64"
      },
-     "execution_count": 104,
+     "execution_count": 101,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -3776,7 +3685,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 105,
+   "execution_count": 102,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:50.829907900Z",
@@ -3791,7 +3700,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 106,
+   "execution_count": 103,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:51.049321400Z",
@@ -3832,7 +3741,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 107,
+   "execution_count": 104,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:51.270758300Z",
@@ -3844,7 +3753,7 @@
      "data": {
       "text/plain": "0.0561016822006403"
      },
-     "execution_count": 107,
+     "execution_count": 104,
      "metadata": {},
      "output_type": "execute_result"
     },
@@ -3883,7 +3792,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 108,
+   "execution_count": 105,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:51.270758300Z",
@@ -3899,7 +3808,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 109,
+   "execution_count": 106,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:51.534025900Z",
@@ -3911,7 +3820,7 @@
      "data": {
       "text/plain": "0.05870838788931646"
      },
-     "execution_count": 109,
+     "execution_count": 106,
      "metadata": {},
      "output_type": "execute_result"
     },
@@ -3955,7 +3864,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 110,
+   "execution_count": 107,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:51.535023300Z",
@@ -3967,7 +3876,7 @@
      "data": {
       "text/plain": "{'steps': [('deseasonalize', Deseasonalizer(model='multiplicative', sp=12)),\n  ('detrend', Detrender(forecaster=PolynomialTrendForecaster())),\n  ('forecast',\n   RecursiveTabularRegressionForecaster(estimator=KNeighborsRegressor(),\n                                        window_length=15))],\n 'deseasonalize': Deseasonalizer(model='multiplicative', sp=12),\n 'detrend': Detrender(forecaster=PolynomialTrendForecaster()),\n 'forecast': RecursiveTabularRegressionForecaster(estimator=KNeighborsRegressor(),\n                                      window_length=15),\n 'deseasonalize__model': 'multiplicative',\n 'deseasonalize__sp': 12,\n 'detrend__forecaster__degree': 1,\n 'detrend__forecaster__regressor': None,\n 'detrend__forecaster__with_intercept': True,\n 'detrend__forecaster': PolynomialTrendForecaster(),\n 'detrend__model': 'additive',\n 'forecast__estimator__algorithm': 'auto',\n 'forecast__estimator__leaf_size': 30,\n 'forecast__estimator__metric': 'minkowski',\n 'forecast__estimator__metric_params': None,\n 'forecast__estimator__n_jobs': None,\n 'forecast__estimator__n_neighbors': 5,\n 'forecast__estimator__p': 2,\n 'forecast__estimator__weights': 'uniform',\n 'forecast__estimator': KNeighborsRegressor(),\n 'forecast__pooling': 'local',\n 'forecast__transformers': None,\n 'forecast__window_length': 15}"
      },
-     "execution_count": 110,
+     "execution_count": 107,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -3998,7 +3907,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 111,
+   "execution_count": 108,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:51.535023300Z",
@@ -4018,7 +3927,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 112,
+   "execution_count": 142,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:51.535023300Z",
@@ -4048,7 +3957,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 113,
+   "execution_count": 143,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:52.403701600Z",
@@ -4060,7 +3969,7 @@
      "data": {
       "text/plain": "0.16607972017556033"
      },
-     "execution_count": 113,
+     "execution_count": 143,
      "metadata": {},
      "output_type": "execute_result"
     },
@@ -4089,7 +3998,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 114,
+   "execution_count": 111,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:52.407690200Z",
@@ -4101,7 +4010,7 @@
      "data": {
       "text/plain": "{'window_length': 7}"
      },
-     "execution_count": 114,
+     "execution_count": 111,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -4119,7 +4028,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 115,
+   "execution_count": 112,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:52.439634200Z",
@@ -4132,7 +4041,7 @@
       "text/plain": "RecursiveTabularRegressionForecaster(estimator=KNeighborsRegressor(),\n                                     window_length=7)",
       "text/html": "<style>#sk-container-id-5 {color: black;background-color: white;}#sk-container-id-5 pre{padding: 0;}#sk-container-id-5 div.sk-toggleable {background-color: white;}#sk-container-id-5 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-5 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-5 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-5 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-5 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-5 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-5 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-5 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-5 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-5 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-5 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-5 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-5 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-5 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-5 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-5 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-5 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-5 div.sk-item {position: relative;z-index: 1;}#sk-container-id-5 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-5 div.sk-item::before, #sk-container-id-5 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-5 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-5 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-5 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-5 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-5 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-5 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-5 div.sk-label-container {text-align: center;}#sk-container-id-5 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-5 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-5\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>RecursiveTabularRegressionForecaster(estimator=KNeighborsRegressor(),\n                                     window_length=7)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-5\" type=\"checkbox\" ><label for=\"sk-estimator-id-5\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">RecursiveTabularRegressionForecaster</label><div class=\"sk-toggleable__content\"><pre>RecursiveTabularRegressionForecaster(estimator=KNeighborsRegressor(),\n                                     window_length=7)</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-6\" type=\"checkbox\" ><label for=\"sk-estimator-id-6\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">estimator: KNeighborsRegressor</label><div class=\"sk-toggleable__content\"><pre>KNeighborsRegressor()</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-7\" type=\"checkbox\" ><label for=\"sk-estimator-id-7\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">KNeighborsRegressor</label><div class=\"sk-toggleable__content\"><pre>KNeighborsRegressor()</pre></div></div></div></div></div></div></div></div></div></div>"
      },
-     "execution_count": 115,
+     "execution_count": 112,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -4154,7 +4063,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 116,
+   "execution_count": 113,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:52.465536100Z",
@@ -4174,7 +4083,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 117,
+   "execution_count": 114,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:52.643061400Z",
@@ -4194,7 +4103,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 118,
+   "execution_count": 115,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:58.959177Z",
@@ -4206,7 +4115,7 @@
      "data": {
       "text/plain": "0.13988948769413537"
      },
-     "execution_count": 118,
+     "execution_count": 115,
      "metadata": {},
      "output_type": "execute_result"
     },
@@ -4228,7 +4137,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 119,
+   "execution_count": 116,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:58.977129Z",
@@ -4240,7 +4149,7 @@
      "data": {
       "text/plain": "{'estimator__n_neighbors': 2, 'window_length': 12}"
      },
-     "execution_count": 119,
+     "execution_count": 116,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -4258,7 +4167,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 120,
+   "execution_count": 117,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:25:58.998074Z",
@@ -4283,7 +4192,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 121,
+   "execution_count": 118,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:26:06.047228700Z",
@@ -4295,7 +4204,7 @@
      "data": {
       "text/plain": "0.14493362646957736"
      },
-     "execution_count": 121,
+     "execution_count": 118,
      "metadata": {},
      "output_type": "execute_result"
     },
@@ -4337,7 +4246,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 122,
+   "execution_count": 119,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:26:06.051246700Z",
@@ -4346,12 +4255,12 @@
    },
    "outputs": [],
    "source": [
-    "from aeon.performance_metrics.forecasting import MeanSquaredError"
+    "from aeon.performance_metrics.forecasting import mean_squared_error"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 123,
+   "execution_count": 120,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:26:06.079142400Z",
@@ -4360,7 +4269,7 @@
    },
    "outputs": [],
    "source": [
-    "mse = MeanSquaredError()\n",
+    "mse = mean_squared_error\n",
     "\n",
     "param_grid = {\"window_length\": [7, 12, 15]}\n",
     "\n",
@@ -4379,7 +4288,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 124,
+   "execution_count": 121,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:26:12.524912Z",
@@ -4389,10 +4298,10 @@
    "outputs": [
     {
      "data": {
-      "text/plain": "   mean_test_MeanSquaredError  mean_fit_time  mean_pred_time  \\\n0                 2600.750255       0.063548        0.002128   \n1                 1134.999053       0.081391        0.002259   \n2                 1285.133614       0.098068        0.002919   \n\n                  params  rank_test_MeanSquaredError  \n0   {'window_length': 7}                         3.0  \n1  {'window_length': 12}                         1.0  \n2  {'window_length': 15}                         2.0  ",
-      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>mean_test_MeanSquaredError</th>\n      <th>mean_fit_time</th>\n      <th>mean_pred_time</th>\n      <th>params</th>\n      <th>rank_test_MeanSquaredError</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>2600.750255</td>\n      <td>0.063548</td>\n      <td>0.002128</td>\n      <td>{'window_length': 7}</td>\n      <td>3.0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1134.999053</td>\n      <td>0.081391</td>\n      <td>0.002259</td>\n      <td>{'window_length': 12}</td>\n      <td>1.0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>1285.133614</td>\n      <td>0.098068</td>\n      <td>0.002919</td>\n      <td>{'window_length': 15}</td>\n      <td>2.0</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
+      "text/plain": "   mean_test_mean_squared_error  mean_fit_time  mean_pred_time  \\\n0                   2600.750255       0.072074        0.002209   \n1                   1134.999053       0.097644        0.002344   \n2                   1285.133614       0.117036        0.003169   \n\n                  params  rank_test_mean_squared_error  \n0   {'window_length': 7}                           3.0  \n1  {'window_length': 12}                           1.0  \n2  {'window_length': 15}                           2.0  ",
+      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>mean_test_mean_squared_error</th>\n      <th>mean_fit_time</th>\n      <th>mean_pred_time</th>\n      <th>params</th>\n      <th>rank_test_mean_squared_error</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>2600.750255</td>\n      <td>0.072074</td>\n      <td>0.002209</td>\n      <td>{'window_length': 7}</td>\n      <td>3.0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1134.999053</td>\n      <td>0.097644</td>\n      <td>0.002344</td>\n      <td>{'window_length': 12}</td>\n      <td>1.0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>1285.133614</td>\n      <td>0.117036</td>\n      <td>0.003169</td>\n      <td>{'window_length': 15}</td>\n      <td>2.0</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
      },
-     "execution_count": 124,
+     "execution_count": 121,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -4429,7 +4338,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 125,
+   "execution_count": 122,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:26:12.575774800Z",
@@ -4445,7 +4354,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 126,
+   "execution_count": 123,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:26:12.580790200Z",
@@ -4464,7 +4373,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 127,
+   "execution_count": 124,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:26:12.581787800Z",
@@ -4477,7 +4386,7 @@
       "text/plain": "MultiplexForecaster(forecasters=[('naive', NaiveForecaster()),\n                                 ('ets',\n                                  ExponentialSmoothing(sp=12, trend='add'))],\n                    selected_forecaster='naive')",
       "text/html": "<style>#sk-container-id-6 {color: black;background-color: white;}#sk-container-id-6 pre{padding: 0;}#sk-container-id-6 div.sk-toggleable {background-color: white;}#sk-container-id-6 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-6 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-6 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-6 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-6 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-6 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-6 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-6 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-6 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-6 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-6 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-6 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-6 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-6 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-6 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-6 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-6 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-6 div.sk-item {position: relative;z-index: 1;}#sk-container-id-6 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-6 div.sk-item::before, #sk-container-id-6 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-6 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-6 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-6 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-6 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-6 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-6 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-6 div.sk-label-container {text-align: center;}#sk-container-id-6 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-6 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-6\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>MultiplexForecaster(forecasters=[(&#x27;naive&#x27;, NaiveForecaster()),\n                                 (&#x27;ets&#x27;,\n                                  ExponentialSmoothing(sp=12, trend=&#x27;add&#x27;))],\n                    selected_forecaster=&#x27;naive&#x27;)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-8\" type=\"checkbox\" checked><label for=\"sk-estimator-id-8\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">MultiplexForecaster</label><div class=\"sk-toggleable__content\"><pre>MultiplexForecaster(forecasters=[(&#x27;naive&#x27;, NaiveForecaster()),\n                                 (&#x27;ets&#x27;,\n                                  ExponentialSmoothing(sp=12, trend=&#x27;add&#x27;))],\n                    selected_forecaster=&#x27;naive&#x27;)</pre></div></div></div></div></div>"
      },
-     "execution_count": 127,
+     "execution_count": 124,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -4489,7 +4398,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 128,
+   "execution_count": 125,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:26:12.656568200Z",
@@ -4502,7 +4411,7 @@
       "text/plain": "MultiplexForecaster(forecasters=[('naive', NaiveForecaster()),\n                                 ('ets',\n                                  ExponentialSmoothing(sp=12, trend='add'))],\n                    selected_forecaster='ets')",
       "text/html": "<style>#sk-container-id-7 {color: black;background-color: white;}#sk-container-id-7 pre{padding: 0;}#sk-container-id-7 div.sk-toggleable {background-color: white;}#sk-container-id-7 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-7 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-7 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-7 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-7 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-7 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-7 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-7 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-7 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-7 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-7 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-7 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-7 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-7 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-7 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-7 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-7 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-7 div.sk-item {position: relative;z-index: 1;}#sk-container-id-7 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-7 div.sk-item::before, #sk-container-id-7 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-7 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-7 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-7 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-7 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-7 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-7 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-7 div.sk-label-container {text-align: center;}#sk-container-id-7 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-7 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-7\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>MultiplexForecaster(forecasters=[(&#x27;naive&#x27;, NaiveForecaster()),\n                                 (&#x27;ets&#x27;,\n                                  ExponentialSmoothing(sp=12, trend=&#x27;add&#x27;))],\n                    selected_forecaster=&#x27;ets&#x27;)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-9\" type=\"checkbox\" checked><label for=\"sk-estimator-id-9\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">MultiplexForecaster</label><div class=\"sk-toggleable__content\"><pre>MultiplexForecaster(forecasters=[(&#x27;naive&#x27;, NaiveForecaster()),\n                                 (&#x27;ets&#x27;,\n                                  ExponentialSmoothing(sp=12, trend=&#x27;add&#x27;))],\n                    selected_forecaster=&#x27;ets&#x27;)</pre></div></div></div></div></div>"
      },
-     "execution_count": 128,
+     "execution_count": 125,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -4523,7 +4432,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 129,
+   "execution_count": 126,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:26:12.656568200Z",
@@ -4540,7 +4449,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 130,
+   "execution_count": 127,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:26:12.656568200Z",
@@ -4562,7 +4471,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 131,
+   "execution_count": 128,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:26:15.116982100Z",
@@ -4574,7 +4483,7 @@
      "data": {
       "text/plain": "0.19886711926999853"
      },
-     "execution_count": 131,
+     "execution_count": 128,
      "metadata": {},
      "output_type": "execute_result"
     },
@@ -4603,7 +4512,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 132,
+   "execution_count": 129,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:26:15.121969800Z",
@@ -4615,7 +4524,7 @@
      "data": {
       "text/plain": "{'selected_forecaster': 'naive'}"
      },
-     "execution_count": 132,
+     "execution_count": 129,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -4626,7 +4535,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 133,
+   "execution_count": 130,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:26:15.122966600Z",
@@ -4639,7 +4548,7 @@
       "text/plain": "MultiplexForecaster(forecasters=[('naive', NaiveForecaster()),\n                                 ('ets',\n                                  ExponentialSmoothing(sp=12, trend='add'))],\n                    selected_forecaster='naive')",
       "text/html": "<style>#sk-container-id-8 {color: black;background-color: white;}#sk-container-id-8 pre{padding: 0;}#sk-container-id-8 div.sk-toggleable {background-color: white;}#sk-container-id-8 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-8 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-8 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-8 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-8 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-8 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-8 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-8 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-8 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-8 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-8 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-8 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-8 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-8 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-8 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-8 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-8 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-8 div.sk-item {position: relative;z-index: 1;}#sk-container-id-8 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-8 div.sk-item::before, #sk-container-id-8 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-8 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-8 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-8 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-8 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-8 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-8 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-8 div.sk-label-container {text-align: center;}#sk-container-id-8 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-8 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-8\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>MultiplexForecaster(forecasters=[(&#x27;naive&#x27;, NaiveForecaster()),\n                                 (&#x27;ets&#x27;,\n                                  ExponentialSmoothing(sp=12, trend=&#x27;add&#x27;))],\n                    selected_forecaster=&#x27;naive&#x27;)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-10\" type=\"checkbox\" checked><label for=\"sk-estimator-id-10\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">MultiplexForecaster</label><div class=\"sk-toggleable__content\"><pre>MultiplexForecaster(forecasters=[(&#x27;naive&#x27;, NaiveForecaster()),\n                                 (&#x27;ets&#x27;,\n                                  ExponentialSmoothing(sp=12, trend=&#x27;add&#x27;))],\n                    selected_forecaster=&#x27;naive&#x27;)</pre></div></div></div></div></div>"
      },
-     "execution_count": 133,
+     "execution_count": 130,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -4669,7 +4578,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 134,
+   "execution_count": 131,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2023-08-25T09:26:15.272566Z",
@@ -4693,13 +4602,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 135,
-   "metadata": {
-    "ExecuteTime": {
-     "end_time": "2023-08-25T09:26:15.320438700Z",
-     "start_time": "2023-08-25T09:26:15.112992900Z"
-    }
-   },
+   "execution_count": 132,
    "outputs": [],
    "source": [
     "# create pipeline\n",
@@ -4722,23 +4625,20 @@
     "    \"forecaster__strategy\": [\"drift\", \"mean\", \"last\"],\n",
     "}\n",
     "gscv = ForecastingGridSearchCV(forecaster=pipe, param_grid=param_grid, cv=cv, n_jobs=-1)"
-   ]
+   ],
+   "metadata": {
+    "collapsed": false
+   }
   },
   {
    "cell_type": "code",
-   "execution_count": 136,
-   "metadata": {
-    "ExecuteTime": {
-     "end_time": "2023-08-25T09:26:17.085195500Z",
-     "start_time": "2023-08-25T09:26:15.119974200Z"
-    }
-   },
+   "execution_count": 133,
    "outputs": [
     {
      "data": {
       "text/plain": "0.1299046419013891"
      },
-     "execution_count": 136,
+     "execution_count": 133,
      "metadata": {},
      "output_type": "execute_result"
     },
@@ -4756,47 +4656,42 @@
     "y_pred = gscv.predict(fh)\n",
     "plot_series(y_train, y_test, y_pred, labels=[\"y_train\", \"y_test\", \"y_pred\"])\n",
     "mean_absolute_percentage_error(y_test, y_pred, symmetric=False)"
-   ]
+   ],
+   "metadata": {
+    "collapsed": false
+   }
   },
   {
    "cell_type": "markdown",
-   "metadata": {},
    "source": [
     "###  3.4.3 Simple ensembling strategies\n",
     "\n",
     "TODO - contributions in this section are appreciated"
-   ]
+   ],
+   "metadata": {
+    "collapsed": false
+   }
   },
   {
    "cell_type": "code",
-   "execution_count": 137,
-   "metadata": {
-    "ExecuteTime": {
-     "end_time": "2023-08-25T09:26:17.085195500Z",
-     "start_time": "2023-08-25T09:26:17.076735100Z"
-    }
-   },
+   "execution_count": 134,
    "outputs": [],
    "source": [
     "from aeon.forecasting.compose import EnsembleForecaster"
-   ]
+   ],
+   "metadata": {
+    "collapsed": false
+   }
   },
   {
    "cell_type": "code",
-   "execution_count": 138,
-   "metadata": {
-    "scrolled": true,
-    "ExecuteTime": {
-     "end_time": "2023-08-25T09:26:17.419302100Z",
-     "start_time": "2023-08-25T09:26:17.081205800Z"
-    }
-   },
+   "execution_count": 135,
    "outputs": [
     {
      "data": {
       "text/plain": "0.1661796803565588"
      },
-     "execution_count": 138,
+     "execution_count": 135,
      "metadata": {},
      "output_type": "execute_result"
     },
@@ -4825,28 +4720,27 @@
     "y_pred = forecaster.predict(fh)\n",
     "plot_series(y_train, y_test, y_pred, labels=[\"y_train\", \"y_test\", \"y_pred\"])\n",
     "mean_absolute_percentage_error(y_test, y_pred, symmetric=False)"
-   ]
+   ],
+   "metadata": {
+    "collapsed": false
+   }
   },
   {
    "cell_type": "markdown",
-   "metadata": {},
    "source": [
     "### 3.4.4 Prediction weighted ensembles and hedge ensembles\n",
     "\n",
     "For model evaluation, we sometimes want to evaluate multiple forecasts, using temporal cross-validation with a sliding window over the test data. For this purpose, we can leverage the forecasters from the `online_forecasting` module which use a composite forecaster, `PredictionWeightedEnsemble`, to keep track of the loss accumulated by each forecaster and create a prediction weighted by the predictions of the most \"accurate\" forecasters.\n",
     "\n",
     "Note that the forecasting task is changed: we make 35 predictions since we need the first prediction to help update the weights, we do not predict 36 steps ahead."
-   ]
+   ],
+   "metadata": {
+    "collapsed": false
+   }
   },
   {
    "cell_type": "code",
-   "execution_count": 139,
-   "metadata": {
-    "ExecuteTime": {
-     "end_time": "2023-08-25T09:26:17.427280700Z",
-     "start_time": "2023-08-25T09:26:17.373425300Z"
-    }
-   },
+   "execution_count": 136,
    "outputs": [],
    "source": [
     "from aeon.forecasting.online_learning import (\n",
@@ -4854,51 +4748,49 @@
     "    OnlineEnsembleForecaster,\n",
     ")\n",
     "from aeon.performance_metrics.forecasting import mean_squared_error"
-   ]
+   ],
+   "metadata": {
+    "collapsed": false
+   }
   },
   {
    "cell_type": "markdown",
-   "metadata": {},
    "source": [
     "First we need to initialize a `PredictionWeightedEnsembler` that will keep track of the loss accumulated by each forecaster and define which loss function we would like to use."
-   ]
+   ],
+   "metadata": {
+    "collapsed": false
+   }
   },
   {
    "cell_type": "code",
-   "execution_count": 140,
-   "metadata": {
-    "ExecuteTime": {
-     "end_time": "2023-08-25T09:26:17.427280700Z",
-     "start_time": "2023-08-25T09:26:17.378411700Z"
-    }
-   },
+   "execution_count": 137,
    "outputs": [],
    "source": [
     "hedge_expert = NormalHedgeEnsemble(n_estimators=3, loss_func=mean_squared_error)"
-   ]
+   ],
+   "metadata": {
+    "collapsed": false
+   }
   },
   {
    "cell_type": "markdown",
-   "metadata": {},
    "source": [
     "We can then create the forecaster by defining the individual forecasters and specifying the `PredictionWeightedEnsembler` we are using. Then by fitting our forecasters and performing updates and prediction with the `update_predict` function, we get:"
-   ]
+   ],
+   "metadata": {
+    "collapsed": false
+   }
   },
   {
    "cell_type": "code",
-   "execution_count": 141,
-   "metadata": {
-    "ExecuteTime": {
-     "end_time": "2023-08-25T09:26:18.790666800Z",
-     "start_time": "2023-08-25T09:26:17.383398500Z"
-    }
-   },
+   "execution_count": 138,
    "outputs": [
     {
      "data": {
       "text/plain": "0.0978975689038194"
      },
-     "execution_count": 141,
+     "execution_count": 138,
      "metadata": {},
      "output_type": "execute_result"
     },
@@ -4925,11 +4817,13 @@
     "y_pred = forecaster.update_predict_single(y_test)\n",
     "plot_series(y_train, y_test, y_pred, labels=[\"y_train\", \"y_test\", \"y_pred\"])\n",
     "mean_absolute_percentage_error(y_test, y_pred, symmetric=False)"
-   ]
+   ],
+   "metadata": {
+    "collapsed": false
+   }
   },
   {
    "cell_type": "markdown",
-   "metadata": {},
    "source": [
     "## 4. Extension guide - implementing your own forecaster\n",
     "\n",
@@ -4954,11 +4848,13 @@
     "* Create a pull request that contains only the new estimators (and their inheritance tree, if it's not just one class), as well as the automated tests as described above.\n",
     "* In the pull request, describe the estimator and optimally provide a publication or other technical reference for the strategy it implements.\n",
     "* Before making the pull request, ensure that you have all necessary permissions to contribute the code to a permissive license (BSD-3) open source project."
-   ]
+   ],
+   "metadata": {
+    "collapsed": false
+   }
   },
   {
    "cell_type": "markdown",
-   "metadata": {},
    "source": [
     "## 5. Summary\n",
     "\n",
@@ -4972,7 +4868,10 @@
     "## Useful resources\n",
     "* For a good introduction to forecasting, see [Hyndman, Rob J., and George Athanasopoulos. Forecasting: principles and practice. OTexts, 2018](https://otexts.com/fpp2/).\n",
     "* For comparative benchmarking studies/forecasting competitions, see the [M4 competition](https://www.sciencedirect.com/science/article/pii/S0169207019301128) and the [M5 competition](https://www.kaggle.com/c/m5-forecasting-accuracy/overview)."
-   ]
+   ],
+   "metadata": {
+    "collapsed": false
+   }
   }
  ],
  "metadata": {


Test patch:
diff --git a/aeon/benchmarking/tests/test_forecasting.py b/aeon/benchmarking/tests/test_forecasting.py
index 6c7239d8..7fd5c679 100644
--- a/aeon/benchmarking/tests/test_forecasting.py
+++ b/aeon/benchmarking/tests/test_forecasting.py
@@ -6,7 +6,7 @@ import pytest
 from aeon.benchmarking import forecasting
 from aeon.forecasting.model_selection import ExpandingWindowSplitter
 from aeon.forecasting.naive import NaiveForecaster
-from aeon.performance_metrics.forecasting import MeanSquaredPercentageError
+from aeon.performance_metrics.forecasting import mean_squared_percentage_error
 from aeon.utils.validation._dependencies import _check_soft_dependencies
 
 
@@ -30,7 +30,7 @@ def test_forecastingbenchmark(tmp_path):
         step_length=1,
         fh=1,
     )
-    benchmark.add_task(data_loader_simple, cv_splitter, [MeanSquaredPercentageError()])
+    benchmark.add_task(data_loader_simple, cv_splitter, [mean_squared_percentage_error])
 
     results_file = tmp_path / "results.csv"
     results_df = benchmark.run(results_file)
@@ -54,10 +54,10 @@ def test_forecastingbenchmark(tmp_path):
         columns=[
             "validation_id",
             "model_id",
-            "MeanSquaredPercentageError_fold_0_test",
-            "MeanSquaredPercentageError_fold_1_test",
-            "MeanSquaredPercentageError_mean",
-            "MeanSquaredPercentageError_std",
+            "mean_squared_percentage_error_fold_0_test",
+            "mean_squared_percentage_error_fold_1_test",
+            "mean_squared_percentage_error_mean",
+            "mean_squared_percentage_error_std",
         ],
     )
 
diff --git a/aeon/forecasting/compose/tests/test_multiplex.py b/aeon/forecasting/compose/tests/test_multiplex.py
index 7ea7d5fa..a3c748fd 100644
--- a/aeon/forecasting/compose/tests/test_multiplex.py
+++ b/aeon/forecasting/compose/tests/test_multiplex.py
@@ -22,7 +22,7 @@ from aeon.utils.validation.forecasting import check_scoring
 def _score_forecasters(forecasters, cv, y):
     """Will evaluate all the forecasters on y and return the name of best."""
     scoring = check_scoring(None)
-    scoring_name = f"test_{scoring.name}"
+    scoring_name = f"test_{scoring.__name__}"
     score = None
     for name, forecaster in forecasters:
         results = evaluate(forecaster, cv, y)
diff --git a/aeon/forecasting/model_evaluation/tests/test_evaluate.py b/aeon/forecasting/model_evaluation/tests/test_evaluate.py
index 4fd76d44..2eb11eb6 100644
--- a/aeon/forecasting/model_evaluation/tests/test_evaluate.py
+++ b/aeon/forecasting/model_evaluation/tests/test_evaluate.py
@@ -28,8 +28,8 @@ from aeon.forecasting.model_selection import (
 from aeon.forecasting.naive import NaiveForecaster
 from aeon.forecasting.tests import TEST_FHS, TEST_STEP_LENGTHS_INT
 from aeon.performance_metrics.forecasting import (
-    MeanAbsolutePercentageError,
-    MeanAbsoluteScaledError,
+    mean_absolute_percentage_error,
+    mean_absolute_scaled_error,
 )
 from aeon.testing.test_config import PR_TESTING
 from aeon.testing.utils.data_gen import _make_hierarchical, make_forecasting_problem
@@ -38,13 +38,13 @@ from aeon.utils.validation._dependencies import _check_soft_dependencies
 if PR_TESTING:
     BACKENDS = [None]
     CV = [SlidingWindowSplitter]
-    SCORING = [MeanAbsolutePercentageError(symmetric=True)]
+    SCORING = [mean_absolute_percentage_error]
     LENGTHS = [7]
     STRAT = ["update"]
 else:
     BACKENDS = [None, "dask", "loky", "threading"]
     CV = [SlidingWindowSplitter, ExpandingWindowSplitter]
-    SCORING = [MeanAbsolutePercentageError(symmetric=True), MeanAbsoluteScaledError()]
+    SCORING = [mean_absolute_percentage_error, mean_absolute_scaled_error]
     LENGTHS = [7, 10]
     STRAT = ["refit", "update"]
 
@@ -58,7 +58,7 @@ def _check_evaluate_output(out, cv, y, scoring):
         "fit_time",
         "len_train_window",
         "pred_time",
-        f"test_{scoring.name}",
+        f"test_{scoring.__name__}",
     }
 
     # Check number of rows against number of splits.
@@ -124,7 +124,7 @@ def test_evaluate_common_configs(
     _check_evaluate_output(out, cv, y, scoring)
 
     # check scoring
-    actual = out.loc[:, f"test_{scoring.name}"]
+    actual = out.loc[:, f"test_{scoring.__name__}"]
 
     n_splits = cv.get_n_splits(y)
     expected = np.empty(n_splits)
@@ -147,13 +147,13 @@ def test_scoring_list(return_data):
         y=y,
         cv=cv,
         scoring=[
-            MeanAbsolutePercentageError(symmetric=True),
-            MeanAbsoluteScaledError(),
+            mean_absolute_percentage_error,
+            mean_absolute_scaled_error,
         ],
         return_data=return_data,
     )
-    assert "test_MeanAbsolutePercentageError" in out.columns
-    assert "test_MeanAbsoluteScaledError" in out.columns
+    assert "test_mean_absolute_percentage_error" in out.columns
+    assert "test_mean_absolute_scaled_error" in out.columns
     if return_data:
         assert "y_pred" in out.columns
         assert "y_train" in out.columns
@@ -171,7 +171,7 @@ def test_evaluate_initial_window():
     forecaster = NaiveForecaster()
     fh = 1
     cv = SlidingWindowSplitter(fh=fh, initial_window=initial_window)
-    scoring = MeanAbsolutePercentageError(symmetric=True)
+    scoring = mean_absolute_percentage_error
     out = evaluate(
         forecaster=forecaster, y=y, cv=cv, strategy="update", scoring=scoring
     )
@@ -179,7 +179,7 @@ def test_evaluate_initial_window():
     assert out.loc[0, "len_train_window"] == initial_window
 
     # check scoring
-    actual = out.loc[0, f"test_{scoring.name}"]
+    actual = out.loc[0, f"test_{scoring.__name__}"]
     train, test = next(cv.split(y))
     f = forecaster.clone()
     f.fit(y.iloc[train], fh=fh)
@@ -192,12 +192,12 @@ def test_evaluate_no_exog_against_with_exog():
     y, X = load_longley()
     forecaster = DirectReductionForecaster(LinearRegression())
     cv = SlidingWindowSplitter()
-    scoring = MeanAbsolutePercentageError(symmetric=True)
+    scoring = mean_absolute_percentage_error
 
     out_exog = evaluate(forecaster, cv, y, X=X, scoring=scoring)
     out_no_exog = evaluate(forecaster, cv, y, X=None, scoring=scoring)
 
-    scoring_name = f"test_{scoring.name}"
+    scoring_name = f"test_{scoring.__name__}"
     assert np.all(out_exog[scoring_name] != out_no_exog[scoring_name])
 
 
@@ -233,9 +233,9 @@ def test_evaluate_error_score(error_score, return_data, strategy, backend):
                 backend=backend,
             )
         if isinstance(error_score, type(np.nan)):
-            assert results["test_MeanAbsolutePercentageError"].isna().sum() > 0
+            assert results["test_mean_absolute_percentage_error"].isna().sum() > 0
         if error_score == 1000:
-            assert results["test_MeanAbsolutePercentageError"].max() == 1000
+            assert results["test_mean_absolute_percentage_error"].max() == 1000
     if error_score == "raise":
         with pytest.raises(Exception):  # noqa: B017
             evaluate(
@@ -266,7 +266,7 @@ def test_evaluate_hierarchical(backend):
 
     forecaster = DirectReductionForecaster(LinearRegression())
     cv = SlidingWindowSplitter()
-    scoring = MeanAbsolutePercentageError(symmetric=True)
+    scoring = mean_absolute_percentage_error
     out_exog = evaluate(
         forecaster, cv, y, X=X, scoring=scoring, error_score="raise", backend=backend
     )
@@ -274,5 +274,5 @@ def test_evaluate_hierarchical(backend):
         forecaster, cv, y, X=None, scoring=scoring, error_score="raise", backend=backend
     )
 
-    scoring_name = f"test_{scoring.name}"
+    scoring_name = f"test_{scoring.__name__}"
     assert np.all(out_exog[scoring_name] != out_no_exog[scoring_name])
diff --git a/aeon/forecasting/model_selection/tests/test_tune.py b/aeon/forecasting/model_selection/tests/test_tune.py
index 6acd9fcf..4780c500 100644
--- a/aeon/forecasting/model_selection/tests/test_tune.py
+++ b/aeon/forecasting/model_selection/tests/test_tune.py
@@ -20,8 +20,8 @@ from aeon.forecasting.naive import NaiveForecaster
 from aeon.forecasting.tests import TEST_N_ITERS, TEST_OOS_FHS, TEST_WINDOW_LENGTHS_INT
 from aeon.forecasting.trend import PolynomialTrendForecaster
 from aeon.performance_metrics.forecasting import (
-    MeanAbsolutePercentageError,
-    MeanSquaredError,
+    mean_absolute_percentage_error,
+    mean_squared_error,
 )
 from aeon.testing.test_config import PR_TESTING
 from aeon.testing.utils.data_gen import _make_hierarchical
@@ -41,11 +41,11 @@ PIPE_GRID = {
 }
 
 if PR_TESTING:
-    TEST_METRICS = [MeanAbsolutePercentageError(symmetric=True)]
+    TEST_METRICS = [mean_absolute_percentage_error]
     ERROR_SCORES = [1000]
     GRID = [(NAIVE, NAIVE_GRID)]
 else:
-    TEST_METRICS = [MeanAbsolutePercentageError(symmetric=True), MeanSquaredError()]
+    TEST_METRICS = [mean_absolute_percentage_error, mean_squared_error]
     ERROR_SCORES = [np.nan, "raise", 1000]
     GRID = [(NAIVE, NAIVE_GRID), (PIPE, PIPE_GRID)]
 
@@ -61,12 +61,12 @@ def _get_expected_scores(forecaster, cv, param_grid, y, X, scoring):
         f = forecaster.clone()
         f.set_params(**params)
         out = evaluate(f, cv, y, X=X, scoring=scoring)
-        scores[i] = out.loc[:, f"test_{scoring.name}"].mean()
+        scores[i] = out.loc[:, f"test_{scoring.__name__}"].mean()
     return scores
 
 
 def _check_cv(forecaster, tuner, cv, param_grid, y, X, scoring):
-    actual = tuner.cv_results_[f"mean_test_{scoring.name}"]
+    actual = tuner.cv_results_[f"mean_test_{scoring.__name__}"]
 
     expected = _get_expected_scores(forecaster, cv, param_grid, y, X, scoring)
     np.testing.assert_array_equal(actual, expected)
diff --git a/aeon/performance_metrics/tests/test_stats.py b/aeon/performance_metrics/tests/test_stats.py
index 7eef938f..4e1d05e8 100644
--- a/aeon/performance_metrics/tests/test_stats.py
+++ b/aeon/performance_metrics/tests/test_stats.py
@@ -1,4 +1,4 @@
-"""Tests for the critical difference diagram maker."""
+"""Tests for the stats and p-values computation."""
 
 import os
 
diff --git a/aeon/transformations/tests/test_multiplexer.py b/aeon/transformations/tests/test_multiplexer.py
index cfb3dbac..9ffd1b6b 100644
--- a/aeon/transformations/tests/test_multiplexer.py
+++ b/aeon/transformations/tests/test_multiplexer.py
@@ -53,7 +53,7 @@ def test_multiplex_transformer_alone():
 def _find_best_transformer(forecaster, transformers, cv, y):
     """Evaluate all the transformers on y and return the name of best."""
     scoring = check_scoring(None)
-    scoring_name = f"test_{scoring.name}"
+    scoring_name = f"test_{scoring.__name__}"
     score = None
     for name, transformer in transformers:
         test_transformer = clone(transformer)