Repository: sktime/sktime
Pull request #8420 (2025-06-22 09:23:28)

[ENH] faster `MovingBlockBootstrapTransformer` tests, improved docstring
Minor improvements to `MovingBlockBootstrapTransformer`:

* faster test cases
* improved docstring with more visible defaults


Patch:
diff --git a/sktime/benchmarking/classification.py b/sktime/benchmarking/classification.py
new file mode 100644
index 000000000..4a1b4156f
--- /dev/null
+++ b/sktime/benchmarking/classification.py
@@ -0,0 +1,382 @@
+"""Benchmarking for classification estimators."""
+
+__author__ = ["jgyasu"]
+__all__ = ["ClassificationBenchmark"]
+
+import logging
+import warnings
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from typing import Any, Optional, Union
+
+import pandas as pd
+
+from sktime.base import BaseEstimator
+from sktime.benchmarking._benchmarking_dataclasses import (
+    FoldResults,
+    ResultObject,
+    TaskObject,
+)
+from sktime.benchmarking._storage_handlers import get_storage_backend
+from sktime.benchmarking._utils import _check_id_format
+from sktime.benchmarking.benchmarks import BaseBenchmark
+from sktime.classification.base import BaseClassifier
+from sktime.classification.model_evaluation import evaluate
+from sktime.utils.unique_str import _make_strings_unique
+
+
+@dataclass
+class _BenchmarkingResults:
+    """Results of a benchmarking run.
+
+    Parameters
+    ----------
+    results : list of ResultObject
+        The results of the benchmarking run.
+    """
+
+    path: str
+    results: list[ResultObject] = field(default_factory=list)
+
+    def __post_init__(self):
+        """Save the results to a file."""
+        self.storage_backend = get_storage_backend(self.path)
+        self.results = self.storage_backend(self.path).load()
+
+    def save(self):
+        """Save the results to a file."""
+        self.storage_backend(self.path).save(self.results)
+
+    def contains(self, task_id: str, model_id: str):
+        """
+        Check if the results contain a specific task and model.
+
+        Parameters
+        ----------
+        task_id : str
+            The task ID.
+        model_id : str
+            The model ID.
+        """
+        return any(
+            [
+                result.task_id == task_id and result.model_id == model_id
+                for result in self.results
+            ]
+        )
+
+    def to_dataframe(self):
+        """Convert the results to a pandas DataFrame."""
+        results = []
+        for result in self.results:
+            results.append(result.to_dataframe())
+
+        df = pd.concat(results, axis=0, ignore_index=True)
+        df["runtime_secs"] = df["pred_time_mean"] + df["fit_time_mean"]
+        return df
+
+
+class _SktimeRegistry:
+    """Register an entity by ID.
+
+    IDs should remain stable over time and should be guaranteed to resolve to
+    the same entity dynamics (or be desupported).
+    """
+
+    def __init__(self, entity_id_format: str = ""):
+        self.entity_id_format = entity_id_format
+        self.entities = {}
+
+    def register(self, entity_id, entity: Union[BaseEstimator, TaskObject]):
+        """Register an entity.
+
+        Parameters
+        ----------
+        entity_id: str
+            A unique entity ID.
+        entry_point: Callable or str
+            The python entrypoint of the entity class. Should be one of:
+            - the string path to the python object (e.g.module.name:factory_func, or
+                module.name:Class)
+            - the python object (class or factory) itself
+        deprecated: Bool, optional (default=False)
+            Flag to denote whether this entity should be skipped in validation runs
+            and considered deprecated and replaced by a more recent/better model
+        nondeterministic: Bool, optional (default=False)
+            Whether this entity is non-deterministic even after seeding
+        kwargs: Dict, optional (default=None)
+            kwargs to pass to the entity entry point when instantiating the entity.
+        """
+        entity_id_unique = _make_strings_unique(list(self.entities.keys()), entity_id)
+        _check_id_format(self.entity_id_format, entity_id_unique)
+        if entity_id != entity_id_unique:
+            warnings.warn(
+                message=f"Entity with ID [id={entity_id}] already registered, "
+                + "new id is {entity_id_unique}",
+                category=UserWarning,
+                stacklevel=2,
+            )
+        self.entities[entity_id_unique] = entity
+
+
+class ClassificationBenchmark(BaseBenchmark):
+    """Classification benchmark.
+
+    Run a series of classifiers against a series of tasks defined via dataset loaders,
+    cross validation splitting strategies and performance metrics, and return results as
+    a df (as well as saving to file).
+
+    Parameters
+    ----------
+    id_format: str, optional (default=None)
+        A regex used to enforce task/estimator ID to match a certain format
+
+    backend : string, by default "None".
+        Parallelization backend to use for runs.
+
+        - "None": executes loop sequentally, simple list comprehension
+        - "loky", "multiprocessing" and "threading": uses ``joblib.Parallel`` loops
+        - "joblib": custom and 3rd party ``joblib`` backends, e.g., ``spark``
+        - "dask": uses ``dask``, requires ``dask`` package in environment
+        - "dask_lazy": same as "dask",
+        but changes the return to (lazy) ``dask.dataframe.DataFrame``.
+        - "ray": uses ``ray``, requires ``ray`` package in environment
+
+        Recommendation: Use "dask" or "loky" for parallel evaluate.
+        "threading" is unlikely to see speed ups due to the GIL and the
+        serialization backend (``cloudpickle``) for "dask" and "loky" is
+        generally more robust than the standard ``pickle`` library used
+        in "multiprocessing".
+
+    backend_params : dict, optional
+        additional parameters passed to the backend as config.
+        Directly passed to ``utils.parallel.parallelize``.
+        Valid keys depend on the value of ``backend``:
+
+        - "None": no additional parameters, ``backend_params`` is ignored
+        - "loky", "multiprocessing" and "threading": default ``joblib`` backends
+        any valid keys for ``joblib.Parallel`` can be passed here, e.g., ``n_jobs``,
+        with the exception of ``backend`` which is directly controlled by
+        ``backend``. If ``n_jobs`` is not passed, it will default to ``-1``, other
+        parameters will default to ``joblib`` defaults.
+        - "joblib": custom and 3rd party ``joblib`` backends, e.g., ``spark``.
+        any valid keys for ``joblib.Parallel`` can be passed here, e.g., ``n_jobs``,
+        ``backend`` must be passed as a key of ``backend_params`` in this case.
+        If ``n_jobs`` is not passed, it will default to ``-1``, other parameters
+        will default to ``joblib`` defaults.
+        - "dask": any valid keys for ``dask.compute`` can be passed,
+        e.g., ``scheduler``
+
+        - "ray": The following keys can be passed:
+
+            - "ray_remote_args": dictionary of valid keys for ``ray.init``
+            - "shutdown_ray": bool, default=True; False prevents ``ray`` from shutting
+                down after parallelization.
+            - "logger_name": str, default="ray"; name of the logger to use.
+            - "mute_warnings": bool, default=False; if True, suppresses warnings
+
+    return_data : bool, optional (default=False)
+        Whether to return the prediction and the ground truth data in the results.
+    """
+
+    def __init__(
+        self,
+        id_format: Optional[str] = None,
+        backend=None,
+        backend_params=None,
+        return_data=False,
+    ):
+        super().__init__(id_format)
+        self.backend = backend
+        self.backend_params = backend_params
+        self.estimators = _SktimeRegistry(id_format)
+        self.tasks = _SktimeRegistry(id_format)
+        self.return_data = return_data
+
+    def add_estimator(
+        self,
+        estimator: BaseEstimator,
+        estimator_id: Optional[str] = None,
+    ):
+        """Register an estimator to the benchmark.
+
+        Parameters
+        ----------
+        estimator : Dict, List or BaseEstimator object
+            Estimator to add to the benchmark.
+            If Dict, keys are estimator_ids used to customise identifier ID
+            and values are estimators.
+            If List, each element is an estimator. estimator_ids are generated
+            automatically using the estimator's class name.
+        estimator_id : str, optional (default=None)
+            Identifier for estimator. If none given then uses estimator's class name.
+        """
+        if isinstance(estimator, dict):
+            for key, value in estimator.items():
+                self.add_estimator(value, key)
+        elif isinstance(estimator, list):
+            for value in estimator:
+                self.add_estimator(value)
+        else:
+            self._add_estimator(estimator, estimator_id)
+
+    def _add_estimator(
+        self,
+        estimator: BaseEstimator,
+        estimator_id: Optional[str] = None,
+    ):
+        """Register an estimator to the benchmark.
+
+        Parameters
+        ----------
+        estimator : Dict, List or BaseEstimator object
+            Estimator to add to the benchmark.
+            If Dict, keys are estimator_ids used to customise identifier ID
+            and values are estimators.
+            If List, each element is an estimator. estimator_ids are generated
+            automatically using the estimator's class name.
+        estimator_id : str, optional (default=None)
+            Identifier for estimator. If none given then uses estimator's class name.
+        """
+        estimator = estimator.clone()
+        if estimator_id is None:
+            estimator_id = estimator.__class__.__name__
+        self.estimators.register(entity_id=estimator_id, entity=estimator)
+
+    def _add_task(
+        self,
+        task_id: str,
+        task: TaskObject,
+    ):
+        """Register a task to the benchmark."""
+        self.tasks.register(
+            entity_id=task_id,
+            entity=task,
+        )
+
+    def add_task(
+        self,
+        dataset_loader: Union[Callable, tuple],
+        cv_splitter: Any,
+        scorers: list,
+        task_id: Optional[str] = None,
+        error_score: str = "raise",
+    ):
+        """Register a classification task to the benchmark.
+
+        Parameters
+        ----------
+        data : Union[Callable, tuple]
+            Can be
+            - a function which returns a dataset, like from `sktime.datasets`.
+            - a tuple contianing two data container that are sktime comptaible.
+            - single data container that is sktime compatible (only endogenous data).
+        cv_splitter : BaseSplitter object
+            Splitter used for generating validation folds.
+        scorers : a list of BaseMetric objects
+            Each BaseMetric output will be included in the results.
+        task_id : str, optional (default=None)
+            Identifier for the benchmark task. If none given then uses dataset loader
+            name combined with cv_splitter class name.
+        error_score : "raise" or numeric, default=np.nan
+            Value to assign to the score if an exception occurs in estimator fitting.
+            If set to "raise", the exception is raised. If a numeric value is given,
+            FitFailedWarning is raised.
+
+        Returns
+        -------
+        A dictionary of benchmark results for that classifier
+        """
+        if task_id is None:
+            if hasattr(dataset_loader, "__name__"):
+                task_id = (
+                    f"[dataset={dataset_loader.__name__}]"
+                    + f"_[cv_splitter={cv_splitter.__class__.__name__}]"
+                )
+            else:
+                task_id = f"_[cv_splitter={cv_splitter.__class__.__name__}]"
+        task_kwargs = {
+            "data": dataset_loader,
+            "cv_splitter": cv_splitter,
+            "scorers": scorers,
+            "error_score": error_score,
+        }
+        self._add_task(
+            task_id,
+            TaskObject(**task_kwargs),
+        )
+
+    def _run(self, results_path: str, force_rerun: Union[str, list[str]] = "none"):
+        """
+        Run the benchmarking for all tasks and estimators.
+
+        Parameters
+        ----------
+        results_path : str
+            Path to save the results to.
+        force_rerun : Union[str, list[str]], optional (default="none")
+            If "none", will skip validation if results already exist.
+            If "all", will run validation for all tasks and models.
+            If list of str, will run validation for tasks and models in list.
+        """
+        results = _BenchmarkingResults(path=results_path)
+
+        for task_id, task in self.tasks.entities.items():
+            for estimator_id, estimator in self.estimators.entities.items():
+                if results.contains(task_id, estimator_id) and (
+                    force_rerun == "none"
+                    or (
+                        isinstance(force_rerun, list)
+                        and estimator_id not in force_rerun
+                    )
+                ):
+                    logging.info(
+                        f"Skipping validation - model: "
+                        f"{task_id} - {estimator_id}"
+                        ", as found prior result in results."
+                    )
+                    continue
+
+                logging.info(f"Running validation - model: {task_id} - {estimator_id}")
+                folds = self._run_validation(task, estimator)
+                results.results.append(
+                    ResultObject(
+                        task_id=task_id,
+                        model_id=estimator_id,
+                        folds=folds,
+                    )
+                )
+
+        results.save()
+        return results.to_dataframe()
+
+    def _run_validation(self, task: TaskObject, estimator: BaseClassifier):
+        cv_splitter = task.cv_splitter
+        scorers = task.scorers
+        X, y = task.get_y_X()
+        scores_df = evaluate(
+            classifier=estimator,
+            y=y,
+            X=X,
+            cv=cv_splitter,
+            scoring=scorers,
+            backend=self.backend,
+            backend_params=self.backend_params,
+            error_score=task.error_score,
+            return_data=self.return_data,
+        )
+
+        folds = {}
+        for ix, row in scores_df.iterrows():
+            scores = {}
+            for scorer in scorers:
+                scores[scorer.__name__] = row["test_" + scorer.__name__]
+            scores["fit_time"] = row["fit_time"]
+            scores["pred_time"] = row["pred_time"]
+            if self.return_data:
+                folds[ix] = FoldResults(
+                    scores, row["y_test"], row["y_pred"], row["y_train"]
+                )
+            else:
+                folds[ix] = FoldResults(scores)
+        return folds
diff --git a/sktime/forecasting/compose/_bagging.py b/sktime/forecasting/compose/_bagging.py
index c47a756ea..79b8aed40 100644
--- a/sktime/forecasting/compose/_bagging.py
+++ b/sktime/forecasting/compose/_bagging.py
@@ -401,7 +401,7 @@ class BaggingForecaster(BaseForecaster):
         from sktime.transformations.bootstrap import MovingBlockBootstrapTransformer
         from sktime.utils.dependencies import _check_soft_dependencies
 
-        mbb = MovingBlockBootstrapTransformer(block_length=6)
+        mbb = MovingBlockBootstrapTransformer(block_length=6, n_series=3)
         fcst = YfromX.create_test_instance()
         params = [{"bootstrap_transformer": mbb, "forecaster": fcst}]
 
diff --git a/sktime/forecasting/dynamic_factor.py b/sktime/forecasting/dynamic_factor.py
index 559ec8dd2..f9b2fb32c 100644
--- a/sktime/forecasting/dynamic_factor.py
+++ b/sktime/forecasting/dynamic_factor.py
@@ -130,7 +130,7 @@ class DynamicFactor(_StatsModelsAdapter):
         # python_dependencies: "statsmodels" - inherited from _StatsModelsAdapter
         # estimator type
         # --------------
-        "scitype:y": "multivariate",
+        "scitype:y": "both",
         "ignores-exogeneous-X": False,
         "capability:missing_values": True,
         "y_inner_mtype": "pd.DataFrame",
@@ -210,7 +210,7 @@ class DynamicFactor(_StatsModelsAdapter):
 
         Returns
         -------
-        y_pred : pd.Series
+        y_pred : pd.DataFrame
             Returns series of predicted values.
         """
         # statsmodels requires zero-based indexing starting at the
@@ -219,6 +219,11 @@ class DynamicFactor(_StatsModelsAdapter):
 
         y_pred = self._fitted_forecaster.predict(start=start, end=end, exog=X)
 
+        # if y is univariate, we duplicated the column in fit,
+        # so now we need to revert this duplication
+        if self._was_univariate:
+            y_pred = y_pred.iloc[:, [0]]
+
         # statsmodels forecasts all periods from start to end of forecasting
         # horizon, but only return given time points in forecasting horizon
 
@@ -284,6 +289,12 @@ class DynamicFactor(_StatsModelsAdapter):
 
             y_pred = model.get_forecast(steps=steps, exog=X).conf_int(alpha=alpha)
 
+            # if y is univariate, we duplicated the column in fit,
+            # so now we need to revert this duplication
+            # subste to first two columns as "lower" and "upper"
+            if self._was_univariate:
+                y_pred = y_pred.iloc[:, [0, 1]]
+
             y_pred = y_pred.iloc[ix]
 
             y_pred.rename(
@@ -295,8 +306,12 @@ class DynamicFactor(_StatsModelsAdapter):
         # concatenate the predictions for different values of alpha
         predictions_df = pd.concat(df_list, axis=1)
 
+        ynames = model.data.ynames
+        if self._was_univariate:
+            ynames = [ynames[0]]
+
         # all the code below is just boilerplate for getting the correct columns
-        mod_var_list = list(map(lambda x: str(x).replace(" ", ""), model.data.ynames))
+        mod_var_list = list(map(lambda x: str(x).replace(" ", ""), ynames))
 
         rename_list = [
             var_name + " " + str(coverage) + " " + bound
@@ -315,25 +330,12 @@ class DynamicFactor(_StatsModelsAdapter):
         ]
 
         predictions_df = predictions_df[final_ord]
-        cols = [col_name.split(" ") for col_name in predictions_df.columns]
 
-        predictions_df_2 = pd.DataFrame(
-            predictions_df.values, columns=pd.MultiIndex.from_tuples(cols)
-        )
+        final_columns = self._get_columns("predict_interval", coverage=coverage_list)
+        predictions_df = pd.DataFrame(predictions_df.values, columns=final_columns)
+        predictions_df.index = fh.to_absolute_index(self.cutoff)
 
-        final_columns = [
-            [col_name, float(coverage), bound]
-            for col_name in model.data.ynames
-            for coverage in predictions_df_2.columns.get_level_values(1).unique()
-            for bound in predictions_df_2.columns.get_level_values(2).unique()
-        ]
-
-        predictions_df_3 = pd.DataFrame(
-            predictions_df_2.values, columns=pd.MultiIndex.from_tuples(final_columns)
-        )
-        predictions_df_3.index = fh.to_absolute_index(self.cutoff)
-
-        return predictions_df_3
+        return predictions_df
 
     def _fit_forecaster(self, y, X=None):
         """Fit to training data.
@@ -349,6 +351,14 @@ class DynamicFactor(_StatsModelsAdapter):
             DynamicFactor as _DynamicFactor,
         )
 
+        # if y is single column DataFrame, duplicate the column to make it multivariate
+        if y.shape[1] == 1:
+            y = pd.concat([y, y], axis=1)
+            y.columns = [f"{y.columns[0]}{i}" for i in ["", "__1"]]
+            self._was_univariate = True
+        else:
+            self._was_univariate = False
+
         self._forecaster = _DynamicFactor(
             endog=y,
             exog=X,
@@ -379,6 +389,14 @@ class DynamicFactor(_StatsModelsAdapter):
             low_memory=self.low_memory,
         )
 
+    def _update(self, y, X=None, update_params=True):
+        """Update used internally in update."""
+        if self._was_univariate:
+            y = pd.concat([y, y], axis=1)
+            y.columns = [f"{y.columns[0]}{i}" for i in ["", "__1"]]
+        super()._update(y, X, update_params=update_params)
+        return self
+
     def summary(self):
         """Get a summary of the fitted forecaster."""
         return self._fitted_forecaster.summary()
diff --git a/sktime/param_est/lag.py b/sktime/param_est/lag.py
index 796fbcea6..25920370d 100644
--- a/sktime/param_est/lag.py
+++ b/sktime/param_est/lag.py
@@ -78,7 +78,7 @@ class ARLagOrderSelector(BaseParamFitter):
     ARLagOrderSelector(...)
     >>> selector.selected_model_
     (3,)
-    >>> selector.ic_value_
+    >>> selector.ic_value_  # doctest: +SKIP
     1369.6963340649502
 
     See Also
diff --git a/sktime/transformations/bootstrap/_mbb.py b/sktime/transformations/bootstrap/_mbb.py
index 0c907426a..a2dc3ee89 100644
--- a/sktime/transformations/bootstrap/_mbb.py
+++ b/sktime/transformations/bootstrap/_mbb.py
@@ -440,22 +440,21 @@ class MovingBlockBootstrapTransformer(BaseTransformer):
 
     Parameters
     ----------
-    n_series : int, optional
-        The number of bootstrapped time series that will be generated, by default 10
-    block_length : int, optional
+    n_series : int, optional, default=10
+        The number of bootstrapped time series that will be generated
+    block_length : int, optional, default = min(2*sp, len(X) - sp)
         The length of the block in the MBB method, by default None.
         If not provided, the following heuristic is used, the block length will the
         minimum between 2*sp and len(X) - sp.
-    sampling_replacement : bool, optional
-        Whether the MBB sample is with or without replacement, by default False.
-    return_actual : bool, optional
-        If True the output will contain the actual time series, by default True.
-        The actual time series will be labelled as "actual"
+    sampling_replacement : bool, optional, default=False
+        Whether the MBB sample is with or without replacement
+    return_actual : bool, optional, default=True
+        If True the output will contain the actual time series.
+        The actual time series will be labelled as "actual".
     random_state : int, np.random.RandomState or None, by default None
         Controls the randomness of the estimator
-    return_indices : bool, optional
-        If True, the output will contain the resampled indices as extra column,
-        by default False.
+    return_indices : bool, optional, default=False.
+        If True, the output will contain the resampled indices as extra column.
 
     See Also
     --------


Test patch:
diff --git a/sktime/benchmarking/tests/test_classification.py b/sktime/benchmarking/tests/test_classification.py
new file mode 100644
index 000000000..aec194427
--- /dev/null
+++ b/sktime/benchmarking/tests/test_classification.py
@@ -0,0 +1,146 @@
+"""Tests for Classification Benchmark."""
+
+__author__ = ["jgyasu"]
+
+import pandas as pd
+from sklearn.metrics import accuracy_score, brier_score_loss
+from sklearn.model_selection import KFold
+
+from sktime.benchmarking.classification import ClassificationBenchmark
+from sktime.classification.distance_based import KNeighborsTimeSeriesClassifier
+from sktime.classification.dummy import DummyClassifier
+from sktime.datasets import load_unit_test
+from sktime.utils._testing.panel import make_classification_problem
+
+
+def test_classification_benchmark(tmp_path):
+    """Test classification benchmark with single estimator and task."""
+    benchmark = ClassificationBenchmark()
+    benchmark.add_estimator(DummyClassifier())
+    scorers = [accuracy_score]
+
+    cv_splitter = KFold(n_splits=3)
+    benchmark.add_task(make_classification_problem, cv_splitter, scorers)
+
+    results_file = tmp_path / "results.csv"
+    results_df = benchmark.run(results_file)
+
+    expected_benchmark_labels = [
+        "accuracy_score_fold_0_test",
+        "accuracy_score_fold_1_test",
+        "accuracy_score_fold_2_test",
+        "accuracy_score_mean",
+        "accuracy_score_std",
+        "fit_time_fold_0_test",
+        "fit_time_fold_1_test",
+        "fit_time_fold_2_test",
+        "fit_time_mean",
+        "fit_time_std",
+        "pred_time_fold_0_test",
+        "pred_time_fold_1_test",
+        "pred_time_fold_2_test",
+        "pred_time_mean",
+        "pred_time_std",
+        "runtime_secs",
+    ]
+
+    result_rows = results_df.T.index.to_list()
+    for metric in expected_benchmark_labels:
+        assert metric in result_rows, f"{metric} not found in result rows"
+
+
+def test_add_list_estimators(tmp_path):
+    """Test adding list of estimators."""
+    benchmark = ClassificationBenchmark()
+
+    estimators = [DummyClassifier(), KNeighborsTimeSeriesClassifier()]
+
+    benchmark.add_estimator(estimators)
+    scorers = [accuracy_score, brier_score_loss]
+
+    cv_splitter = KFold(n_splits=3)
+    benchmark.add_task(make_classification_problem, cv_splitter, scorers)
+
+    results_file = tmp_path / "results.csv"
+    results_df = benchmark.run(results_file)
+
+    pd.testing.assert_series_equal(
+        pd.Series(
+            ["DummyClassifier", "KNeighborsTimeSeriesClassifier"], name="model_id"
+        ),
+        results_df["model_id"],
+    )
+
+
+def test_add_dict_estimators(tmp_path):
+    """Test adding dict of estimators."""
+    benchmark = ClassificationBenchmark()
+
+    estimators = {"D": DummyClassifier(), "KN": KNeighborsTimeSeriesClassifier()}
+
+    benchmark.add_estimator(estimators)
+    scorers = [accuracy_score, brier_score_loss]
+
+    cv_splitter = KFold(n_splits=3)
+    benchmark.add_task(make_classification_problem, cv_splitter, scorers)
+
+    results_file = tmp_path / "results.csv"
+    results_df = benchmark.run(results_file)
+
+    pd.testing.assert_series_equal(
+        pd.Series(["D", "KN"], name="model_id"),
+        results_df["model_id"],
+    )
+
+
+def test_add_estimator_twice(tmp_path):
+    """Test adding the same estimator twice."""
+    benchmark = ClassificationBenchmark()
+    benchmark.add_estimator(DummyClassifier())
+    benchmark.add_estimator(DummyClassifier())
+    scorers = [accuracy_score]
+
+    cv_splitter = KFold(n_splits=3)
+    benchmark.add_task(make_classification_problem, cv_splitter, scorers)
+
+    results_file = tmp_path / "results.csv"
+    results_df = benchmark.run(results_file)
+
+    pd.testing.assert_series_equal(
+        pd.Series(["DummyClassifier", "DummyClassifier_2"], name="model_id"),
+        results_df["model_id"],
+    )
+
+    msg = "add_estimator does not register all estimators."
+    assert len(benchmark.estimators.entities) == 2, msg
+
+
+def test_add_multiple_task(tmp_path):
+    """Test adding multiple tasks for benchmarking."""
+    benchmark = ClassificationBenchmark()
+    benchmark.add_estimator(DummyClassifier())
+
+    dataset_loaders = [make_classification_problem, load_unit_test]
+    cv_splitter = KFold(n_splits=3)
+    scorers = [accuracy_score, brier_score_loss]
+
+    for dataset_loader in dataset_loaders:
+        benchmark.add_task(
+            dataset_loader,
+            cv_splitter,
+            scorers,
+        )
+
+    results_file = tmp_path / "results.csv"
+    results_df = benchmark.run(results_file)
+
+    pd.testing.assert_series_equal(
+        pd.Series(
+            [
+                "[dataset=make_classification_problem]_[cv_splitter=KFold]",
+                "[dataset=load_unit_test]_[cv_splitter=KFold]",
+            ],
+            name="validation_id",
+        ),
+        results_df["validation_id"],
+    )
diff --git a/sktime/tests/test_switch.py b/sktime/tests/test_switch.py
index 341718f9e..7fc8e1feb 100644
--- a/sktime/tests/test_switch.py
+++ b/sktime/tests/test_switch.py
@@ -229,7 +229,7 @@ def _run_test_for_class(cls):
         test_classes = get_test_classes_for_obj(cls)
         return any(is_class_changed(x) for x in test_classes)
 
-    def _is_impacted_by_pyproject_change(cls):
+    def _is_impacted_by_pyproject_change(cls, include_core_deps=False):
         """Check if the dep specifications of cls have changed, return bool."""
         from packaging.requirements import Requirement
 
@@ -244,6 +244,17 @@ def _run_test_for_class(cls):
         cls_reqs = _flatten_list(cls_reqs)
         package_deps = [Requirement(req).name for req in cls_reqs]
 
+        if include_core_deps:
+            CORE_DEPENDENCIES = [
+                "scikit-base",
+                "scikit-learn",
+                "scipy",
+                "numpy",
+                "pandas",
+                "scikit-base",
+            ]
+            package_deps += CORE_DEPENDENCIES
+
         return any(x in PACKAGE_REQ_CHANGED for x in package_deps)
 
     # Condition 1:
@@ -260,16 +271,19 @@ def _run_test_for_class(cls):
     # run the test if and only if at least one of the conditions 2-4 are met
     # conditions are checked in order to minimize runtime due to git diff etc
 
+    # variable: is cls a core object?
+    cls_is_core = _is_core_object(cls)
+
     # Condition 4:
     # the package requirements for any dependency in pyproject.toml have changed
-    cond4 = _is_impacted_by_pyproject_change(cls)
+    cond4 = _is_impacted_by_pyproject_change(cls, include_core_deps=cls_is_core)
     if cond4:
         return True, "True_pyproject_change"
 
     # Condition 3:
     # if the object is an sktime BaseObject, and one of the test classes
     # covering the class have changed, then run the test
-    if _is_core_object(cls) and _tests_covering_class_changed(cls):
+    if cls_is_core and _tests_covering_class_changed(cls):
         return True, "True_changed_tests"
 
     # Condition 2:
@@ -283,7 +297,7 @@ def _run_test_for_class(cls):
     # Condition 5 (only for core objects):
     # if the object is an sktime BaseObject, and one of the core framework modules
     # datatypes, tests, utils have changed, then run the test
-    if _is_core_object(cls):
+    if cls_is_core:
         FRAMEWORK_MODULES = [
             "sktime.datatypes",
             "sktime.tests._config",