Repository: sktime/sktime

Issue:
[ENH] speed up notebooks and notebook tests
It seems the notebooks in `examples` take 50min to run, all taken together.

I remember times where this was closer to 20min - we should investigate where runtime is spent in the notebooks, and optimally replace by faster examples that illustrate the same point(s).

Good first issue - simply go through the notebooks, run them and check which cells take longest.

There are also a number of more "obscure" notebooks, we may consider removing them, or shortening them, or making the data sets smaller.

Comments:

Patch:
diff --git a/examples/02a_classification_multivariate_cnn.ipynb b/examples/02a_classification_multivariate_cnn.ipynb
index c2a097c52..f4b464240 100644
--- a/examples/02a_classification_multivariate_cnn.ipynb
+++ b/examples/02a_classification_multivariate_cnn.ipynb
@@ -71,7 +71,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "network = CNNClassifier(n_epochs=200, verbose=True)\n",
+    "network = CNNClassifier(n_epochs=50, verbose=True)\n",
     "network.fit(X_train, y_train)\n",
     "network.score(X_test, y_test)"
    ]
@@ -97,7 +97,7 @@
    "outputs": [],
    "source": [
     "param_grid = {\"kernel_size\": [7, 9], \"avg_pool_size\": [3, 5]}\n",
-    "grid = GridSearchCV(network, param_grid=param_grid, cv=5)\n",
+    "grid = GridSearchCV(network, param_grid=param_grid, cv=3)\n",
     "grid.fit(X_train, y_train)\n",
     "\n",
     "print(\"Best cross-validation accuracy: {:.2f}\".format(grid.best_score_))\n",
@@ -133,5 +133,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 2
-
 }
diff --git a/examples/02b_classification_multivariate_lstmfcn.ipynb b/examples/02b_classification_multivariate_lstmfcn.ipynb
index b11ec338b..c7486fc88 100644
--- a/examples/02b_classification_multivariate_lstmfcn.ipynb
+++ b/examples/02b_classification_multivariate_lstmfcn.ipynb
@@ -75,7 +75,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "network = LSTMFCNClassifier(n_epochs=200, verbose=0)\n",
+    "network = LSTMFCNClassifier(n_epochs=65, verbose=0)\n",
     "network.fit(X_train, y_train)\n",
     "network.score(X_test, y_test)"
    ]
diff --git a/examples/02c_classification_multivariate_inceptiontime.ipynb b/examples/02c_classification_multivariate_inceptiontime.ipynb
index 5bf5c433c..6094630fd 100644
--- a/examples/02c_classification_multivariate_inceptiontime.ipynb
+++ b/examples/02c_classification_multivariate_inceptiontime.ipynb
@@ -87,7 +87,7 @@
    },
    "outputs": [],
    "source": [
-    "network = InceptionTimeClassifier(n_epochs=200, verbose=False)\n",
+    "network = InceptionTimeClassifier(n_epochs=75, verbose=False)\n",
     "network.fit(X_train, y_train)\n",
     "network.score(X_test, y_test)"
    ]
@@ -126,9 +126,9 @@
  "metadata": {
   "file_extension": ".py",
   "kernelspec": {
-   "name": "python3",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
-   "display_name": "Python 3 (ipykernel)"
+   "name": "python3"
   },
   "language_info": {
    "codemirror_mode": {
@@ -150,5 +150,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 2
-
 }
diff --git a/examples/dictionary_based_classification.ipynb b/examples/dictionary_based_classification.ipynb
index f65398b23..bb756d52b 100644
--- a/examples/dictionary_based_classification.ipynb
+++ b/examples/dictionary_based_classification.ipynb
@@ -346,7 +346,7 @@
    "source": [
     "# Recommended non-contract TDE parameters\n",
     "tde_u = TemporalDictionaryEnsemble(\n",
-    "    n_parameter_samples=250,\n",
+    "    n_parameter_samples=50,\n",
     "    max_ensemble_size=50,\n",
     "    randomly_selected_params=50,\n",
     "    random_state=47,\n",
@@ -395,7 +395,7 @@
    "source": [
     "# Recommended non-contract TDE parameters\n",
     "tde_mv = TemporalDictionaryEnsemble(\n",
-    "    n_parameter_samples=250,\n",
+    "    n_parameter_samples=50,\n",
     "    max_ensemble_size=50,\n",
     "    randomly_selected_params=50,\n",
     "    random_state=47,\n",
diff --git a/sktime/forecasting/model_selection/_tune.py b/sktime/forecasting/model_selection/_tune.py
index 704121477..e7de352f8 100644
--- a/sktime/forecasting/model_selection/_tune.py
+++ b/sktime/forecasting/model_selection/_tune.py
@@ -2,7 +2,7 @@
 # copyright: sktime developers, BSD-3-Clause License (see LICENSE file)
 """Implements grid search functionality to tune forecasters."""
 
-__author__ = ["mloning"]
+__author__ = ["mloning", "fkiraly", "aiwalter"]
 __all__ = [
     "ForecastingGridSearchCV",
     "ForecastingRandomizedSearchCV",
@@ -11,6 +11,7 @@ __all__ = [
 
 from collections.abc import Sequence
 from typing import Dict, List, Optional, Union
+from warnings import warn
 
 import numpy as np
 import pandas as pd
@@ -36,6 +37,8 @@ class BaseGridSearch(_DelegatedForecaster):
         "capability:pred_int:insample": True,
     }
 
+    # todo 0.24.0: replace all tune_by_variable defaults in this file with False
+    # remove deprecation message in BaseGridSearch.__init__
     def __init__(
         self,
         forecaster,
@@ -50,6 +53,8 @@ class BaseGridSearch(_DelegatedForecaster):
         return_n_best_forecasters=1,
         update_behaviour="full_refit",
         error_score=np.nan,
+        tune_by_instance=False,
+        tune_by_variable=None,
     ):
         self.forecaster = forecaster
         self.cv = cv
@@ -63,13 +68,30 @@ class BaseGridSearch(_DelegatedForecaster):
         self.return_n_best_forecasters = return_n_best_forecasters
         self.update_behaviour = update_behaviour
         self.error_score = error_score
+        self.tune_by_instance = tune_by_instance
+        self.tune_by_variable = tune_by_variable
+
         super().__init__()
+
+        # todo 0.24.0: remove this
+        if tune_by_variable is None:
+            warn(
+                f"in {self.__class__.__name__}, the default for tune_by_variable "
+                "will change from True to False in 0.24.0. "
+                "This will tune one parameter setting for all variables, while "
+                "currently it tunes one parameter per variable. "
+                "In order to maintain the current behaviour, ensure to set "
+                "the parameter tune_by_variable to True explicitly before upgrading "
+                "to version 0.24.0.",
+                DeprecationWarning,
+            )
+            tune_by_variable = True
+
         tags_to_clone = [
             "requires-fh-in-fit",
             "capability:pred_int",
             "capability:pred_int:insample",
             "capability:insample",
-            "scitype:y",
             "ignores-exogeneous-X",
             "handles-missing-data",
             "y_inner_mtype",
@@ -81,6 +103,11 @@ class BaseGridSearch(_DelegatedForecaster):
         self._extend_to_all_scitypes("y_inner_mtype")
         self._extend_to_all_scitypes("X_inner_mtype")
 
+        # this ensures univariate broadcasting over variables
+        # if tune_by_variable is True
+        if tune_by_variable:
+            self.set_tags(**{"scitype:y": "univariate"})
+
     # attribute for _DelegatedForecaster, which then delegates
     #     all non-overridden methods are same as of getattr(self, _delegate_name)
     #     see further details in _DelegatedForecaster docstring
@@ -91,6 +118,11 @@ class BaseGridSearch(_DelegatedForecaster):
 
         Mutates self tag with name `tagname`.
         If no mtypes are present of a time series scitype, adds a pandas based one.
+        If only univariate pandas scitype is present for Series ("pd.Series"),
+        also adds the multivariate one ("pd.DataFrame").
+
+        If tune_by_instance is True, only Series mtypes are added,
+        and potentially present Panel or Hierarchical mtypes are removed.
 
         Parameters
         ----------
@@ -104,12 +136,22 @@ class BaseGridSearch(_DelegatedForecaster):
         if not isinstance(tagval, list):
             tagval = [tagval]
         scitypes = mtype_to_scitype(tagval, return_unique=True)
+        # if no Series mtypes are present, add pd.DataFrame
         if "Series" not in scitypes:
             tagval = tagval + ["pd.DataFrame"]
+        # ensure we have a Series mtype capable of multivariate
+        elif "pd.Series" in tagval and "pd.DataFrame" not in tagval:
+            tagval = ["pd.DataFrame"] + tagval
+        # if no Panel mtypes are present, add pd.DataFrame based one
         if "Panel" not in scitypes:
             tagval = tagval + ["pd-multiindex"]
+        # if no Hierarchical mtypes are present, add pd.DataFrame based one
         if "Hierarchical" not in scitypes:
             tagval = tagval + ["pd_multiindex_hier"]
+
+        if self.tune_by_instance:
+            tagval = [x for x in tagval if mtype_to_scitype(x) == "Series"]
+
         self.set_tags(**{tagname: tagval})
 
     def _get_fitted_params(self):
@@ -418,6 +460,22 @@ class ForecastingGridSearchCV(BaseGridSearch):
         Value to assign to the score if an exception occurs in estimator fitting. If set
         to "raise", the exception is raised. If a numeric value is given,
         FitFailedWarning is raised.
+    tune_by_instance : bool, optional (default=False)
+        Whether to tune parameter by each time series instance separately,
+        in case of Panel or Hierarchical data passed to the tuning estimator.
+        Only applies if time series passed are Panel or Hierarchical.
+        If True, clones of the forecaster will be fit to each instance separately,
+        and are available in fields of the forecasters_ attribute.
+        Has the same effect as applying ForecastByLevel wrapper to self.
+        If False, the same best parameter is selected for all instances.
+    tune_by_variable : bool, optional (default=True)
+        Whether to tune parameter by each time series variable separately,
+        in case of multivariate data passed to the tuning estimator.
+        Only applies if time series passed are strictly multivariate.
+        If True, clones of the forecaster will be fit to each variable separately,
+        and are available in fields of the forecasters_ attribute.
+        Has the same effect as applying ColumnEnsembleForecaster wrapper to self.
+        If False, the same best parameter is selected for all variables.
 
     Attributes
     ----------
@@ -441,6 +499,12 @@ class ForecastingGridSearchCV(BaseGridSearch):
     n_best_scores_: list of float
         The scores of n_best_forecasters_ sorted from best to worst
         score of forecasters
+    forecasters_ : pd.DataFramee
+        DataFrame with all fitted forecasters and their parameters.
+        Only present if tune_by_instance=True or tune_by_variable=True,
+        and at least one of the two is applicable.
+        In this case, the other attributes are not present in self,
+        only in the fields of forecasters_.
 
     Examples
     --------
@@ -520,6 +584,8 @@ class ForecastingGridSearchCV(BaseGridSearch):
         backend="loky",
         update_behaviour="full_refit",
         error_score=np.nan,
+        tune_by_instance=False,
+        tune_by_variable=None,
     ):
         super().__init__(
             forecaster=forecaster,
@@ -534,6 +600,8 @@ class ForecastingGridSearchCV(BaseGridSearch):
             backend=backend,
             update_behaviour=update_behaviour,
             error_score=error_score,
+            tune_by_instance=tune_by_instance,
+            tune_by_variable=tune_by_variable,
         )
         self.param_grid = param_grid
 
@@ -701,6 +769,22 @@ class ForecastingRandomizedSearchCV(BaseGridSearch):
         Value to assign to the score if an exception occurs in estimator fitting. If set
         to "raise", the exception is raised. If a numeric value is given,
         FitFailedWarning is raised.
+    tune_by_instance : bool, optional (default=False)
+        Whether to tune parameter by each time series instance separately,
+        in case of Panel or Hierarchical data passed to the tuning estimator.
+        Only applies if time series passed are Panel or Hierarchical.
+        If True, clones of the forecaster will be fit to each instance separately,
+        and are available in fields of the forecasters_ attribute.
+        Has the same effect as applying ForecastByLevel wrapper to self.
+        If False, the same best parameter is selected for all instances.
+    tune_by_variable : bool, optional (default=True)
+        Whether to tune parameter by each time series variable separately,
+        in case of multivariate data passed to the tuning estimator.
+        Only applies if time series passed are strictly multivariate.
+        If True, clones of the forecaster will be fit to each variable separately,
+        and are available in fields of the forecasters_ attribute.
+        Has the same effect as applying ColumnEnsembleForecaster wrapper to self.
+        If False, the same best parameter is selected for all variables.
 
     Attributes
     ----------
@@ -718,6 +802,12 @@ class ForecastingRandomizedSearchCV(BaseGridSearch):
     n_best_scores_: list of float
         The scores of n_best_forecasters_ sorted from best to worst
         score of forecasters
+    forecasters_ : pd.DataFramee
+        DataFrame with all fitted forecasters and their parameters.
+        Only present if tune_by_instance=True or tune_by_variable=True,
+        and at least one of the two is applicable.
+        In this case, the other attributes are not present in self,
+        only in the fields of forecasters_.
     """
 
     def __init__(
@@ -737,6 +827,8 @@ class ForecastingRandomizedSearchCV(BaseGridSearch):
         backend="loky",
         update_behaviour="full_refit",
         error_score=np.nan,
+        tune_by_instance=False,
+        tune_by_variable=None,
     ):
         super().__init__(
             forecaster=forecaster,
@@ -751,6 +843,8 @@ class ForecastingRandomizedSearchCV(BaseGridSearch):
             backend=backend,
             update_behaviour=update_behaviour,
             error_score=error_score,
+            tune_by_instance=tune_by_instance,
+            tune_by_variable=tune_by_variable,
         )
         self.param_distributions = param_distributions
         self.n_iter = n_iter
@@ -906,6 +1000,22 @@ class ForecastingSkoptSearchCV(BaseGridSearch):
     backend : str, optional (default="loky")
         Specify the parallelisation backend implementation in joblib, where
         "loky" is used by default.
+    tune_by_instance : bool, optional (default=False)
+        Whether to tune parameter by each time series instance separately,
+        in case of Panel or Hierarchical data passed to the tuning estimator.
+        Only applies if time series passed are Panel or Hierarchical.
+        If True, clones of the forecaster will be fit to each instance separately,
+        and are available in fields of the forecasters_ attribute.
+        Has the same effect as applying ForecastByLevel wrapper to self.
+        If False, the same best parameter is selected for all instances.
+    tune_by_variable : bool, optional (default=True)
+        Whether to tune parameter by each time series variable separately,
+        in case of multivariate data passed to the tuning estimator.
+        Only applies if time series passed are strictly multivariate.
+        If True, clones of the forecaster will be fit to each variable separately,
+        and are available in fields of the forecasters_ attribute.
+        Has the same effect as applying ColumnEnsembleForecaster wrapper to self.
+        If False, the same best parameter is selected for all variables.
 
     Attributes
     ----------
@@ -923,6 +1033,12 @@ class ForecastingSkoptSearchCV(BaseGridSearch):
     n_best_scores_: list of float
         The scores of n_best_forecasters_ sorted from best to worst
         score of forecasters
+    forecasters_ : pd.DataFramee
+        DataFrame with all fitted forecasters and their parameters.
+        Only present if tune_by_instance=True or tune_by_variable=True,
+        and at least one of the two is applicable.
+        In this case, the other attributes are not present in self,
+        only in the fields of forecasters_.
 
     Examples
     --------
@@ -982,6 +1098,8 @@ class ForecastingSkoptSearchCV(BaseGridSearch):
         backend: str = "loky",
         update_behaviour: str = "full_refit",
         error_score=np.nan,
+        tune_by_instance=False,
+        tune_by_variable=None,
     ):
         self.param_distributions = param_distributions
         self.n_iter = n_iter
@@ -1001,6 +1119,8 @@ class ForecastingSkoptSearchCV(BaseGridSearch):
             backend=backend,
             update_behaviour=update_behaviour,
             error_score=error_score,
+            tune_by_instance=tune_by_instance,
+            tune_by_variable=tune_by_variable,
         )
 
     def _fit(self, y, X=None, fh=None):


Test patch:
diff --git a/sktime/forecasting/model_selection/tests/test_tune.py b/sktime/forecasting/model_selection/tests/test_tune.py
index 12afc70a7..f374e51a6 100644
--- a/sktime/forecasting/model_selection/tests/test_tune.py
+++ b/sktime/forecasting/model_selection/tests/test_tune.py
@@ -1,9 +1,8 @@
 #!/usr/bin/env python3 -u
 # copyright: sktime developers, BSD-3-Clause License (see LICENSE file)
-"""Test grid search CV."""
+"""Test forecasting tuners."""
 
 __author__ = ["mloning", "fkiraly"]
-__all__ = ["test_gscv", "test_rscv"]
 
 import numpy as np
 import pytest
@@ -81,12 +80,13 @@ def _check_cv(forecaster, tuner, cv, param_grid, y, X, scoring):
     assert param_grid[best_idx].items() <= fitted_params.items()
 
 
-def _create_hierarchical_data():
+def _create_hierarchical_data(n_columns=1):
     y = _make_hierarchical(
         random_state=TEST_RANDOM_SEEDS[0],
         hierarchy_levels=(2, 2),
         min_timepoints=15,
         max_timepoints=15,
+        n_columns=n_columns,
     )
     X = _make_hierarchical(
         random_state=TEST_RANDOM_SEEDS[1],
@@ -97,7 +97,10 @@ def _create_hierarchical_data():
     return y, X
 
 
-NAIVE = NaiveForecaster(strategy="mean")
+# estimator fixtures used for tuning
+# set_tags in NaiveForecaster ensures that it is univariate and broadcasts
+# this is currently the case, but a future improved NaiveForecaster may reduce coverage
+NAIVE = NaiveForecaster(strategy="mean").set_tags(**{"scitype:y": "univariate"})
 NAIVE_GRID = {"window_length": TEST_WINDOW_LENGTHS_INT}
 PIPE = TransformedTargetForecaster(
     [
@@ -126,15 +129,24 @@ ERROR_SCORES = [np.nan, "raise", 1000]
 @pytest.mark.parametrize("scoring", TEST_METRICS)
 @pytest.mark.parametrize("cv", CVs)
 @pytest.mark.parametrize("error_score", ERROR_SCORES)
-def test_gscv(forecaster, param_grid, cv, scoring, error_score):
+@pytest.mark.parametrize("multivariate", [True, False])
+def test_gscv(forecaster, param_grid, cv, scoring, error_score, multivariate):
     """Test ForecastingGridSearchCV."""
-    y, X = load_longley()
+    if multivariate:
+        X, y = load_longley()
+    else:
+        y, X = load_longley()
+
     gscv = ForecastingGridSearchCV(
         forecaster,
         param_grid=param_grid,
         cv=cv,
         scoring=scoring,
         error_score=error_score,
+        # todo 0.24.0: remove this
+        # and/or add a test for tune_by_variable=True
+        # in this case, the forecaster is expeceted to vectorize over columns
+        tune_by_variable=False,
     )
     gscv.fit(y, X)
 
@@ -177,6 +189,7 @@ def test_rscv(forecaster, param_grid, cv, scoring, error_score, n_iter, random_s
         ParameterSampler(param_grid, n_iter, random_state=random_state)
     )
     _check_cv(forecaster, rscv, cv, param_distributions, y, X, scoring)
+    _check_fitted_params_keys(rscv.get_fitted_params())
 
 
 @pytest.mark.skipif(
@@ -189,20 +202,26 @@ def test_rscv(forecaster, param_grid, cv, scoring, error_score, n_iter, random_s
 @pytest.mark.parametrize("scoring", TEST_METRICS)
 @pytest.mark.parametrize("cv", CVs)
 @pytest.mark.parametrize("error_score", ERROR_SCORES)
-def test_gscv_hierarchical(forecaster, param_grid, cv, scoring, error_score):
+@pytest.mark.parametrize("n_cols", [1, 2])
+def test_gscv_hierarchical(forecaster, param_grid, cv, scoring, error_score, n_cols):
     """Test ForecastingGridSearchCV."""
-    y, X = _create_hierarchical_data()
+    y, X = _create_hierarchical_data(n_columns=n_cols)
     gscv = ForecastingGridSearchCV(
         forecaster,
         param_grid=param_grid,
         cv=cv,
         scoring=scoring,
         error_score=error_score,
+        # todo 0.24.0: remove this
+        # and/or add a test for tune_by_variable=True
+        # in this case, the forecaster is expeceted to vectorize over columns
+        tune_by_variable=False,
     )
     gscv.fit(y, X)
 
     param_grid = ParameterGrid(param_grid)
     _check_cv(forecaster, gscv, cv, param_grid, y, X, scoring)
+    _check_fitted_params_keys(gscv.get_fitted_params())
 
 
 @pytest.mark.skipif(
diff --git a/sktime/proba/tests/test_all_distrs.py b/sktime/proba/tests/test_all_distrs.py
index 5e8594495..5808d9d2c 100644
--- a/sktime/proba/tests/test_all_distrs.py
+++ b/sktime/proba/tests/test_all_distrs.py
@@ -72,7 +72,7 @@ class TestAllDistributions(DistributionFixtureGenerator, QuickTester):
         assert (res_panel.index == dummy_panel.index).all()
         assert (res_panel.columns == dummy_panel.columns).all()
 
-    @pytest.mark.parametrize("method", METHODS_SCALAR)
+    @pytest.mark.parametrize("method", METHODS_SCALAR, ids=METHODS_SCALAR)
     def test_methods_scalar(self, estimator_instance, method):
         """Test expected return of scalar methods."""
         if not _has_capability(estimator_instance, method):
@@ -83,7 +83,7 @@ class TestAllDistributions(DistributionFixtureGenerator, QuickTester):
 
         _check_output_format(res, d, method)
 
-    @pytest.mark.parametrize("method", METHODS_X)
+    @pytest.mark.parametrize("method", METHODS_X, ids=METHODS_X)
     def test_methods_x(self, estimator_instance, method):
         """Test expected return of methods that take sample-like argument."""
         if not _has_capability(estimator_instance, method):
@@ -95,7 +95,7 @@ class TestAllDistributions(DistributionFixtureGenerator, QuickTester):
 
         _check_output_format(res, d, method)
 
-    @pytest.mark.parametrize("method", METHODS_P)
+    @pytest.mark.parametrize("method", METHODS_P, ids=METHODS_P)
     def test_methods_p(self, estimator_instance, method):
         """Test expected return of methods that take percentage-like argument."""
         if not _has_capability(estimator_instance, method):
diff --git a/sktime/registry/tests/test_tags.py b/sktime/registry/tests/test_tags.py
new file mode 100644
index 000000000..e3e738163
--- /dev/null
+++ b/sktime/registry/tests/test_tags.py
@@ -0,0 +1,24 @@
+"""Tests for tag register an tag functionality."""
+
+from sktime.registry._tags import ESTIMATOR_TAG_REGISTER
+
+
+def test_tag_register_type():
+    """Test the specification of the tag register. See _tags for specs."""
+    assert isinstance(ESTIMATOR_TAG_REGISTER, list)
+    assert all(isinstance(tag, tuple) for tag in ESTIMATOR_TAG_REGISTER)
+
+    for tag in ESTIMATOR_TAG_REGISTER:
+        assert len(tag) == 4
+        assert isinstance(tag[0], str)
+        assert isinstance(tag[1], (str, list))
+        if isinstance(tag[1], list):
+            assert all(isinstance(x, str) for x in tag[1])
+        assert isinstance(tag[2], (str, tuple))
+        if isinstance(tag[2], tuple):
+            assert len(tag[2]) == 2
+            assert isinstance(tag[2][0], str)
+            assert isinstance(tag[2][1], (list, str))
+            if isinstance(tag[2][1], list):
+                assert all(isinstance(x, str) for x in tag[2][1])
+        assert isinstance(tag[3], str)