Repository: sktime/sktime

Issue:
[ENH] ForecastingBayesianSearchCV, skopt interface
Would it be possible to add support for Bayesian search, e.g. make skopt.BayesSearchCV accessible through ForecastingBayesSearchCV? Similar to how ForecastingGridSearchCV provides an interface to GridSearchCV from sklearn. 

This would be a great help to tune more complex models with large parameter space.


Discussion:
Yes, great idea!

Perhaps we could, in a second step, also add an abstraction step to what search algorithm is being used.
Hi, I'd like to work on this. Just need a quick tip on where to start
thanks, @GigaMoksh!

This would be very similar to `ForecastingGridSearchCV`, that would be a good place to start.

You may also like the "implementing estimators" guide, which has a link to a tutorial video:
https://www.sktime.org/en/stable/developer_guide/add_estimators.html

Resolving patch:
diff --git a/sktime/forecasting/model_selection/__init__.py b/sktime/forecasting/model_selection/__init__.py
--- a/sktime/forecasting/model_selection/__init__.py
+++ b/sktime/forecasting/model_selection/__init__.py
@@ -11,6 +11,7 @@
     "ExpandingWindowSplitter",
     "ForecastingGridSearchCV",
     "ForecastingRandomizedSearchCV",
+    "ForecastingSkoptSearchCV",
 ]
 
 from sktime.forecasting.model_selection._split import (
@@ -23,4 +24,5 @@
 from sktime.forecasting.model_selection._tune import (
     ForecastingGridSearchCV,
     ForecastingRandomizedSearchCV,
+    ForecastingSkoptSearchCV,
 )
diff --git a/sktime/forecasting/model_selection/_tune.py b/sktime/forecasting/model_selection/_tune.py
--- a/sktime/forecasting/model_selection/_tune.py
+++ b/sktime/forecasting/model_selection/_tune.py
@@ -3,9 +3,14 @@
 """Implements grid search functionality to tune forecasters."""
 
 __author__ = ["mloning"]
-__all__ = ["ForecastingGridSearchCV", "ForecastingRandomizedSearchCV"]
+__all__ = [
+    "ForecastingGridSearchCV",
+    "ForecastingRandomizedSearchCV",
+    "ForecastingSkoptSearchCV",
+]
 
 from collections.abc import Sequence
+from typing import Dict, List, Optional, Union
 
 import numpy as np
 import pandas as pd
@@ -16,6 +21,8 @@
 from sktime.exceptions import NotFittedError
 from sktime.forecasting.base._delegate import _DelegatedForecaster
 from sktime.forecasting.model_evaluation import evaluate
+from sktime.forecasting.model_selection._split import BaseSplitter
+from sktime.performance_metrics.base import BaseMetric
 from sktime.utils.validation.forecasting import check_scoring
 
 
@@ -717,3 +724,538 @@ def get_test_params(cls, parameter_set="default"):
         }
 
         return [params, params2]
+
+
+class ForecastingSkoptSearchCV(BaseGridSearch):
+    """Bayesian search over hyperparameters for a forecaster.
+
+    Experimental: This feature is under development and interface may likely to change.
+
+    Parameters
+    ----------
+    forecaster : estimator object.
+        The estimator should implement the sktime or scikit-learn estimator interface.
+        Either the estimator must contain a "score" function,
+        or a scoring function must be passed.
+    cv : cross-validation generator or an iterable
+        Splitter used for generating validation folds.
+        e.g. SlidingWindowSplitter()
+    param_distributions : dict or a list of dict/tuple. See below for details.
+        1. If dict, a dictionary that represents the search space over the parameters of
+        the provided estimator. The keys are parameter names (strings), and the values
+        follows the following format. A list to store categorical parameters and a
+        tuple for integer and real parameters with the following format
+        (int/float, int/float, "prior") e.g (1e-6, 1e-1, "log-uniform").
+        2. If a list of dict, each dictionary corresponds to a parameter space,
+        following the same structure described in case 1 above. the search will be
+        performed sequentially for each parameter space, with the number of samples
+        set to n_iter.
+        3. If a list of tuple, tuple must contain (dict, int) where the int refers to
+        n_iter for that search space. dict must follow the same structure as in case 1.
+        This is useful if you want to perform a search with different number of
+        iterations for each parameter space.
+    n_iter : int, default=10
+        Number of parameter settings that are sampled. n_iter trades
+        off runtime vs quality of the solution. Consider increasing n_points
+        if you want to try more parameter settings in parallel.
+    n_points : int, default=1
+        Number of parameter settings to sample in parallel.
+        If this does not align with n_iter, the last iteration will sample less points
+    scoring : sktime metric object (BaseMetric), or callable, optional (default=None)
+        scoring metric to use in tuning the forecaster
+        if callable, must have signature
+        `(y_true: 1D np.ndarray, y_pred: 1D np.ndarray) -> float`,
+        assuming np.ndarrays being of the same length, and lower being better.
+    optimizer_kwargs: dict, optional
+        Arguments passed to Optimizer to control the bahaviour of the bayesian search.
+        For example, {'base_estimator': 'RF'} would use a Random Forest surrogate
+        instead of the default Gaussian Process. Please refer to the `skopt.Optimizer`
+        documentation for more information.
+    random_state : int, RandomState instance or None, default=None
+        Pseudo random number generator state used for random uniform sampling
+        from lists of possible values instead of scipy.stats distributions.
+        Pass an int for reproducible output across multiple
+        function calls.
+    strategy : {"refit", "update", "no-update_params"}, optional, default="refit"
+        data ingestion strategy in fitting cv, passed to `evaluate` internally
+        defines the ingestion mode when the forecaster sees new data when window expands
+        "refit" = forecaster is refitted to each training window
+        "update" = forecaster is updated with training window data, in sequence provided
+        "no-update_params" = fit to first training window, re-used without fit or update
+    update_behaviour: str, optional, default = "full_refit"
+        one of {"full_refit", "inner_only", "no_update"}
+        behaviour of the forecaster when calling update
+        "full_refit" = both tuning parameters and inner estimator refit on all data seen
+        "inner_only" = tuning parameters are not re-tuned, inner estimator is updated
+        "no_update" = neither tuning parameters nor inner estimator are updated
+    refit : bool, optional (default=True)
+        True = refit the forecaster with the best parameters on the entire data in fit
+        False = best forecaster remains fitted on the last fold in cv
+    verbose : int, optional (default=0)
+    error_score : "raise" or numeric, default=np.nan
+        Value to assign to the score if an exception occurs in estimator fitting. If set
+        to "raise", the exception is raised. If a numeric value is given,
+        FitFailedWarning is raised.
+    return_n_best_forecasters: int, default=1
+        In case the n best forecaster should be returned, this value can be set
+        and the n best forecasters will be assigned to n_best_forecasters_
+    pre_dispatch : str, optional (default='2*n_jobs')
+    n_jobs : int, optional (default=None)
+        Number of jobs to run in parallel.
+        None means 1 unless in a joblib.parallel_backend context.
+        -1 means using all processors.
+    backend : str, optional (default="loky")
+        Specify the parallelisation backend implementation in joblib, where
+        "loky" is used by default.
+
+    Attributes
+    ----------
+    best_index_ : int
+    best_score_: float
+        Score of the best model
+    best_params_ : dict
+        Best parameter values across the parameter grid
+    best_forecaster_ : estimator
+        Fitted estimator with the best parameters
+    cv_results_ : dict
+        Results from grid search cross validation
+    n_best_forecasters_: list of tuples ("rank", <forecaster>)
+        The "rank" is in relation to best_forecaster_
+    n_best_scores_: list of float
+        The scores of n_best_forecasters_ sorted from best to worst
+        score of forecasters
+
+    Examples
+    --------
+    >>> from sktime.datasets import load_shampoo_sales
+    >>> from sktime.forecasting.model_selection import (
+    ...     ExpandingWindowSplitter,
+    ...     ForecastingSkoptSearchCV)
+    >>> from sklearn.ensemble import GradientBoostingRegressor
+    >>> from sktime.forecasting.compose import make_reduction
+    >>> y = load_shampoo_sales()
+    >>> fh = [1,2,3,4]
+    >>> cv = ExpandingWindowSplitter(fh=fh)
+    >>> forecaster = make_reduction(GradientBoostingRegressor(random_state=10))
+    >>> param_distributions = {
+    ...     "estimator__learning_rate" : (1e-4, 1e-1, "log-uniform"),
+    ...     "window_length" : (1, 10, "uniform"),
+    ...     "estimator__criterion" : ["friedman_mse", "squared_error"]}
+    >>> sscv = ForecastingSkoptSearchCV(
+    ...     forecaster=forecaster,
+    ...     param_distributions=param_distributions,
+    ...     cv=cv,
+    ...     n_iter=5,
+    ...     random_state=10)  # doctest: +SKIP
+    >>> sscv.fit(y)  # doctest: +SKIP
+    ForecastingSkoptSearchCV(...)
+    >>> y_pred = sscv.predict(fh)  # doctest: +SKIP
+    """
+
+    _tags = {
+        "scitype:y": "both",
+        "requires-fh-in-fit": False,
+        "handles-missing-data": False,
+        "ignores-exogeneous-X": True,
+        "capability:pred_int": True,
+        "capability:pred_int:insample": True,
+        "python_dependencies": ["numpy<1.24", "scikit-optimize"],
+        "python_version": ">= 3.6",
+        "python_dependencies_alias": {"scikit-optimize": "skopt"},
+    }
+
+    def __init__(
+        self,
+        forecaster,
+        cv: BaseSplitter,
+        param_distributions: Union[Dict, List[Dict]],
+        n_iter: int = 10,
+        n_points: Optional[int] = 1,
+        random_state: Optional[int] = None,
+        scoring: Optional[List[BaseMetric]] = None,
+        optimizer_kwargs: Optional[Dict] = None,
+        strategy: Optional[str] = "refit",
+        n_jobs: Optional[int] = None,
+        refit: bool = True,
+        verbose: int = 0,
+        return_n_best_forecasters: int = 1,
+        pre_dispatch: str = "2*n_jobs",
+        backend: str = "loky",
+        update_behaviour: str = "full_refit",
+        error_score=np.nan,
+    ):
+        self.param_distributions = param_distributions
+        self.n_iter = n_iter
+        self.n_points = n_points
+        self.random_state = random_state
+        self.optimizer_kwargs = optimizer_kwargs
+        super().__init__(
+            forecaster=forecaster,
+            scoring=scoring,
+            strategy=strategy,
+            n_jobs=n_jobs,
+            refit=refit,
+            cv=cv,
+            verbose=verbose,
+            return_n_best_forecasters=return_n_best_forecasters,
+            pre_dispatch=pre_dispatch,
+            backend=backend,
+            update_behaviour=update_behaviour,
+            error_score=error_score,
+        )
+
+    def _fit(self, y, X=None, fh=None):
+        """Run fit with all sets of parameters."""
+        self._check_cv = check_cv(self.cv)
+        self._check_scoring = check_scoring(self.scoring, obj=self)
+        scoring_name = f"test_{self._check_scoring.name}"
+        self._check_search_space(self.param_distributions)
+        self.cv_results_ = pd.DataFrame()
+
+        self._run_search(y, X)
+
+        # Rank results, according to whether greater is better for the given scoring.
+        self.cv_results_[f"rank_{scoring_name}"] = self.cv_results_.loc[
+            :, f"mean_{scoring_name}"
+        ].rank(ascending=self._check_scoring.get_tag("lower_is_better"))
+
+        results = self.cv_results_
+        # Select best parameters.
+        self.best_index_ = results.loc[:, f"rank_{scoring_name}"].argmin()
+        # Raise error if all fits in evaluate failed because all score values are NaN.
+        if self.best_index_ == -1:
+            raise NotFittedError(
+                f"""All fits of forecaster failed,
+                set error_score='raise' to see the exceptions.
+                Failed forecaster: {self.forecaster}"""
+            )
+        self.best_score_ = results.loc[self.best_index_, f"mean_{scoring_name}"]
+        self.best_params_ = results.loc[self.best_index_, "params"]
+        self.best_forecaster_ = self.forecaster.clone().set_params(**self.best_params_)
+
+        # Refit model with best parameters.
+        if self.refit:
+            self.best_forecaster_.fit(y, X, fh)
+
+        # Sort values according to rank
+        results = results.sort_values(
+            by=f"rank_{scoring_name}",
+            ascending=True,
+        )
+
+        # Select n best forecaster
+        self.n_best_forecasters_ = []
+        self.n_best_scores_ = []
+        for i in range(self.return_n_best_forecasters):
+            params = results["params"].iloc[i]
+            rank = results[f"rank_{scoring_name}"].iloc[i]
+            rank = str(int(rank))
+            forecaster = self.forecaster.clone().set_params(**params)
+            # Refit model with best parameters.
+            if self.refit:
+                forecaster.fit(y, X, fh)
+            self.n_best_forecasters_.append((rank, forecaster))
+            # Save score
+            score = results[f"mean_{scoring_name}"].iloc[i]
+            self.n_best_scores_.append(score)
+
+        return self
+
+    def _run_search(self, y, X=None, fh=None):
+        """Search n_iter candidates from param_distributions.
+
+        Parameters
+        ----------
+        y : time series in sktime compatible data container format
+            Target time series to which to fit the forecaster.
+        X : time series in sktime compatible format, optional (default=None)
+            Exogenous variables.
+        fh : int, list, np.array or ForecastingHorizon, optional (default=None)
+        """
+        # check if space is a single dict, convert to list if so
+        param_distributions = self.param_distributions
+        if isinstance(param_distributions, dict):
+            param_distributions = [param_distributions]
+
+        if self.optimizer_kwargs is None:
+            self.optimizer_kwargs_ = {}
+        else:
+            self.optimizer_kwargs_ = dict(self.optimizer_kwargs)
+        self.optimizer_kwargs_["random_state"] = self.random_state
+
+        optimizers = []
+        mappings = []
+        for search_space in param_distributions:
+            if isinstance(search_space, tuple):
+                search_space = search_space[0]
+
+            # hacky approach to handle unhashable type objects
+            if "forecaster" in search_space:
+                forecasters = search_space.get("forecaster")
+                mapping = {
+                    num: estimator for num, estimator in enumerate(forecasters)
+                }  # noqa
+                search_space["forecaster"] = list(mapping.keys())
+                mappings.append(mapping)
+            else:
+                mappings.append(None)
+
+            optimizers.append(self._create_optimizer(search_space))
+        self.optimizers_ = optimizers  # will save the states of the optimizers
+
+        if self.verbose > 0:
+            n_candidates = self.n_iter
+            n_splits = self.cv.get_n_splits(y)
+            print(  # noqa
+                "Fitting {} folds for each of {} candidates,"
+                " totalling {} fits".format(
+                    n_splits, n_candidates, n_candidates * n_splits
+                )
+            )
+
+        # Run sequential search by iterating through each optimizer and evaluates
+        # the search space iteratively until all n_iter candidates are evaluated.
+        for num, (search_space, optimizer) in enumerate(
+            zip(param_distributions, optimizers)
+        ):
+            # if search subspace n_iter is provided, use it otherwise use self.n_iter
+            if isinstance(search_space, tuple):
+                n_iter = search_space[1]
+            else:
+                n_iter = self.n_iter
+
+            # iterations for each search space
+            while n_iter > 0:
+                # when n_iter < n_points points left for evaluation
+                n_points_adjusted = min(n_iter, self.n_points)
+                self._evaluate_step(
+                    y,
+                    X,
+                    optimizer,
+                    n_points=n_points_adjusted,
+                    mapping=mappings[num],
+                )
+                n_iter -= self.n_points
+            # reset n_iter for next search space
+            n_iter = self.n_iter
+
+    def _evaluate_step(self, y, X, optimizer, n_points, mapping=None):
+        """Evaluate a candidate parameter set at each iteration.
+
+        Parameters
+        ----------
+        y : time series in sktime compatible data container format
+            Target time series to which to fit the forecaster.
+        X : time series in sktime compatible format, optional (default=None)
+            Exogenous variables.
+        optimizer : skopt.Optimizer
+            Optimizer instance.
+        n_points : int
+            Number of candidate parameter combination to evaluate at each step.
+            if n_points=2, then the two candidate parameter combinations are evaluated
+            e.g {'sp': 1, 'strategy':'last'} and {'sp': 2, 'strategy': 'mean'}.
+        mapping : dict, optional (default=None)
+            Mapping of forecaster to estimator instance.
+        """
+        from skopt.utils import use_named_args
+
+        # Get a list of dimension parameter space with name from optimizer
+        dimensions = optimizer.space.dimensions
+        test_score_name = f"test_{self._check_scoring.name}"
+
+        @use_named_args(dimensions)  # decorater to convert candidate param list to dict
+        def _fit_and_score(**params):
+            # Clone forecaster.
+            forecaster = self.forecaster.clone()
+
+            # map forecaster back to estimator instance
+            if "forecaster" in params:
+                params["forecaster"] = mapping[params["forecaster"]]
+
+            # Set parameters.
+            forecaster.set_params(**params)
+
+            # Evaluate.
+            out = evaluate(
+                forecaster=forecaster,
+                cv=self._check_cv,
+                y=y,
+                X=X,
+                strategy=self.strategy,
+                scoring=self._check_scoring,
+                error_score=self.error_score,
+            )
+
+            # Filter columns.
+            out = out.filter(
+                items=[test_score_name, "fit_time", "pred_time"],
+                axis=1,
+            )
+
+            # Aggregate results.
+            out = out.mean()
+            out = out.add_prefix("mean_")
+
+            # Add parameters to output table.
+            out["params"] = params
+
+            return out
+
+        parallel = Parallel(
+            n_jobs=self.n_jobs, pre_dispatch=self.pre_dispatch, backend=self.backend
+        )
+
+        candidate_params = optimizer.ask(n_points=n_points)
+        out = parallel(delayed(_fit_and_score)(params) for params in candidate_params)
+
+        # fetch the mean evaluation metrics and feed them back to optimizer
+        results_df = pd.DataFrame(out)
+        # as the optimizer is minimising a score,
+        # we need to negate the score if higher_is_better
+        mean_test_score = results_df["mean_" + test_score_name]
+        if self._check_scoring.get_tag("lower_is_better"):
+            scores = list(mean_test_score)
+        else:
+            scores = list(-mean_test_score)
+        # Update optimizer with evaluation metrics.
+        optimizer.tell(candidate_params, scores)
+        # keep updating the cv_results_ attribute by concatinating the result dataframe
+        self.cv_results_ = pd.concat([self.cv_results_, results_df], ignore_index=True)
+
+        try:
+            assert len(out) >= 1
+        except AssertionError:
+            raise ValueError(
+                "No fits were performed. "
+                "Was the CV iterator empty? "
+                "Were there no candidates?"
+            )
+
+    def _create_optimizer(self, params_space):
+        """Instantiate optimizer for a search parameter space.
+
+        Responsible for initialising optimizer with the correct parameters
+        names and values.
+
+        Parameters
+        ----------
+        params_space : dict
+            Dictionary with parameters names (string) as keys and the values are
+            instances of skopt.space.Dimension (Real, Integer, or Categorical)
+
+        Returns
+        -------
+        optimizer : skopt.Optimizer
+        """
+        from skopt.optimizer import Optimizer
+        from skopt.utils import dimensions_aslist
+
+        kwargs = self.optimizer_kwargs_.copy()
+        # convert params space to a list ordered by the key name
+        kwargs["dimensions"] = dimensions_aslist(params_space)
+        dimensions_name = list(sorted(params_space.keys()))
+        optimizer = Optimizer(**kwargs)
+        # set the name of the dimensions if not set
+        for i in range(len(optimizer.space.dimensions)):
+            if optimizer.space.dimensions[i].name is not None:
+                continue
+            optimizer.space.dimensions[i].name = dimensions_name[i]
+
+        return optimizer
+
+    def _check_search_space(self, search_space):
+        """Check whether the search space argument is correct.
+
+        from skopt.BayesSearchCV._check_search_space
+        """
+        from skopt.space import check_dimension
+
+        if len(search_space) == 0:
+            raise ValueError(
+                "The search_spaces parameter should contain at least one"
+                "non-empty search space, got %s" % search_space
+            )
+
+        # check if space is a single dict, convert to list if so
+        if isinstance(search_space, dict):
+            search_space = [search_space]
+
+        # check if the structure of the space is proper
+        if isinstance(search_space, list):
+            # convert to just a list of dicts
+            dicts_only = []
+
+            # 1. check the case when a tuple of space, n_iter is provided
+            for elem in search_space:
+                if isinstance(elem, tuple):
+                    if len(elem) != 2:
+                        raise ValueError(
+                            "All tuples in list of search spaces should have"
+                            "length 2, and contain (dict, int), got %s" % elem
+                        )
+                    subspace, n_iter = elem
+
+                    if (not isinstance(n_iter, int)) or n_iter < 0:
+                        raise ValueError(
+                            "Number of iterations in search space should be"
+                            "positive integer, got %s in tuple %s " % (n_iter, elem)
+                        )
+
+                    # save subspaces here for further checking
+                    dicts_only.append(subspace)
+                elif isinstance(elem, dict):
+                    dicts_only.append(elem)
+                else:
+                    raise TypeError(
+                        "A search space should be provided as a dict or"
+                        "tuple (dict, int), got %s" % elem
+                    )
+
+            # 2. check all the dicts for correctness of contents
+            for subspace in dicts_only:
+                for params_name, param_value in subspace.items():
+                    if params_name != "forecaster":
+                        check_dimension(param_value)
+        else:
+            raise TypeError(
+                "Search space should be provided as a dict or list of dict,"
+                "got %s" % search_space
+            )
+
+    @classmethod
+    def get_test_params(cls, parameter_set="default"):
+        """Return testing parameter settings for the estimator.
+
+        Parameters
+        ----------
+        parameter_set : str, default="default"
+            Name of the set of test parameters to return, for use in tests. If no
+            special parameters are defined for a value, will return `"default"` set.
+
+        Returns
+        -------
+        params : dict or list of dict
+        """
+        from sktime.forecasting.model_selection._split import SingleWindowSplitter
+        from sktime.forecasting.naive import NaiveForecaster
+        from sktime.forecasting.trend import PolynomialTrendForecaster
+        from sktime.performance_metrics.forecasting import MeanAbsolutePercentageError
+
+        params = {
+            "forecaster": NaiveForecaster(strategy="mean"),
+            "cv": SingleWindowSplitter(fh=1),
+            "param_distributions": {"window_length": [2, 5]},
+            "scoring": MeanAbsolutePercentageError(symmetric=True),
+            "n_iter": 1,
+        }
+
+        params2 = {
+            "forecaster": PolynomialTrendForecaster(),
+            "cv": SingleWindowSplitter(fh=1),
+            "param_distributions": {"degree": [1, 2]},
+            "scoring": MeanAbsolutePercentageError(symmetric=True),
+            "update_behaviour": "inner_only",
+            "n_iter": 1,
+        }
+
+        return [params, params2]


Test patch:
diff --git a/sktime/forecasting/model_selection/tests/test_tune.py b/sktime/forecasting/model_selection/tests/test_tune.py
--- a/sktime/forecasting/model_selection/tests/test_tune.py
+++ b/sktime/forecasting/model_selection/tests/test_tune.py
@@ -12,10 +12,12 @@
 from sktime.datasets import load_airline, load_longley
 from sktime.forecasting.arima import ARIMA
 from sktime.forecasting.compose import TransformedTargetForecaster
+from sktime.forecasting.exp_smoothing import ExponentialSmoothing
 from sktime.forecasting.model_evaluation import evaluate
 from sktime.forecasting.model_selection import (
     ForecastingGridSearchCV,
     ForecastingRandomizedSearchCV,
+    ForecastingSkoptSearchCV,
     SingleWindowSplitter,
     SlidingWindowSplitter,
 )
@@ -33,6 +35,7 @@
 )
 from sktime.performance_metrics.forecasting.probabilistic import CRPS, PinballLoss
 from sktime.transformations.series.detrend import Detrender
+from sktime.transformations.series.impute import Imputer
 from sktime.utils._testing.hierarchical import _make_hierarchical
 from sktime.utils.validation._dependencies import _check_estimator_deps
 
@@ -50,6 +53,12 @@ def _get_expected_scores(forecaster, cv, param_grid, y, X, scoring):
     return scores
 
 
+def _check_fitted_params_keys(fitted_params):
+    # ensure that the best_forecaster and best_score are in fitted_params
+    assert "best_forecaster" in fitted_params.keys()
+    assert "best_score" in fitted_params.keys()
+
+
 def _check_cv(forecaster, tuner, cv, param_grid, y, X, scoring):
     actual = tuner.cv_results_[f"mean_test_{scoring.name}"]
 
@@ -64,6 +73,22 @@ def _check_cv(forecaster, tuner, cv, param_grid, y, X, scoring):
     assert param_grid[best_idx].items() <= fitted_params.items()
 
 
+def _create_hierarchical_data():
+    y = _make_hierarchical(
+        random_state=TEST_RANDOM_SEEDS[0],
+        hierarchy_levels=(2, 2),
+        min_timepoints=20,
+        max_timepoints=20,
+    )
+    X = _make_hierarchical(
+        random_state=TEST_RANDOM_SEEDS[1],
+        hierarchy_levels=(2, 2),
+        min_timepoints=20,
+        max_timepoints=20,
+    )
+    return y, X
+
+
 NAIVE = NaiveForecaster(strategy="mean")
 NAIVE_GRID = {"window_length": TEST_WINDOW_LENGTHS_INT}
 PIPE = TransformedTargetForecaster(
@@ -103,10 +128,7 @@ def test_gscv(forecaster, param_grid, cv, scoring, error_score):
 
     param_grid = ParameterGrid(param_grid)
     _check_cv(forecaster, gscv, cv, param_grid, y, X, scoring)
-
-    fitted_params = gscv.get_fitted_params()
-    assert "best_forecaster" in fitted_params.keys()
-    assert "best_score" in fitted_params.keys()
+    _check_fitted_params_keys(gscv.get_fitted_params())
 
 
 @pytest.mark.parametrize(
@@ -149,13 +171,7 @@ def test_rscv(forecaster, param_grid, cv, scoring, error_score, n_iter, random_s
 @pytest.mark.parametrize("error_score", ERROR_SCORES)
 def test_gscv_hierarchical(forecaster, param_grid, cv, scoring, error_score):
     """Test ForecastingGridSearchCV."""
-    y = _make_hierarchical(
-        random_state=0, hierarchy_levels=(2, 2), min_timepoints=20, max_timepoints=20
-    )
-    X = _make_hierarchical(
-        random_state=42, hierarchy_levels=(2, 2), min_timepoints=20, max_timepoints=20
-    )
-
+    y, X = _create_hierarchical_data()
     gscv = ForecastingGridSearchCV(
         forecaster,
         param_grid=param_grid,
@@ -194,7 +210,85 @@ def test_gscv_proba(cv, scoring, error_score):
 
     param_grid = ParameterGrid(param_grid)
     _check_cv(forecaster, gscv, cv, param_grid, y, None, scoring)
+    _check_fitted_params_keys(gscv.get_fitted_params())
 
-    fitted_params = gscv.get_fitted_params()
-    assert "best_forecaster" in fitted_params.keys()
-    assert "best_score" in fitted_params.keys()
+
+@pytest.mark.skipif(
+    not _check_estimator_deps(ForecastingSkoptSearchCV, severity="none"),
+    reason="skip test if required soft dependency not compatible",
+)
+@pytest.mark.parametrize(
+    "forecaster, param_grid", [(NAIVE, NAIVE_GRID), (PIPE, PIPE_GRID)]
+)
+@pytest.mark.parametrize("scoring", TEST_METRICS)
+@pytest.mark.parametrize("error_score", ERROR_SCORES)
+@pytest.mark.parametrize("cv", CVs)
+@pytest.mark.parametrize("n_iter", TEST_N_ITERS)
+def test_skoptcv(forecaster, param_grid, cv, scoring, error_score, n_iter):
+    """Test ForecastingSkoptSearchCV.
+
+    Tests that ForecastingSkoptSearchCV successfully searches the
+    parameter distributions to identify the best parameter set
+    """
+    # test for forecasting dataset
+    y_forecasting, X_forecasting = load_longley()
+    # test for hierarchical dataset
+    y_hierarchical, X_hierarchical = _create_hierarchical_data()
+
+    datasets = [(y_hierarchical, X_hierarchical), (y_forecasting, X_forecasting)]
+    sscv = ForecastingSkoptSearchCV(
+        forecaster,
+        param_distributions=param_grid,
+        cv=cv,
+        scoring=scoring,
+        error_score=error_score,
+        n_iter=n_iter,
+        random_state=42,
+        n_jobs=-1,
+    )
+    for y, X in datasets:
+        sscv.fit(y, X)
+        param_distributions = list(sscv.cv_results_["params"])
+        _check_cv(forecaster, sscv, cv, param_distributions, y, X, scoring)
+        _check_fitted_params_keys(sscv.get_fitted_params())
+
+
+@pytest.mark.skipif(
+    not _check_estimator_deps(ForecastingSkoptSearchCV, severity="none"),
+    reason="skip test if required soft dependency not compatible",
+)
+def test_skoptcv_multiple_forecaster():
+    """Test ForecastingSkoptSearchCV with multiple forecasters and custom n_iter.
+
+    Other behaviours are tested in test_skoptcv.
+    """
+    params_distributions = [
+        {
+            "forecaster": [NaiveForecaster(sp=12)],
+            "forecaster__strategy": ["drift", "last", "mean"],
+        },  # iterate twice
+        (
+            {
+                "imputer__method": ["mean", "median"],
+                "forecaster": [ExponentialSmoothing(sp=12)],
+                "forecaster__trend": ["add", "mul"],
+            },
+            3,
+        ),  # iterate thrice custom iteration
+    ]
+    cv = CVs[-1]
+    y, X = load_longley()
+    pipe = TransformedTargetForecaster(
+        steps=[("imputer", Imputer()), ("forecaster", NaiveForecaster())]
+    )
+    sscv = ForecastingSkoptSearchCV(
+        forecaster=pipe,
+        param_distributions=params_distributions,
+        cv=cv,
+        n_jobs=-1,
+        random_state=123,
+        n_points=2,
+        n_iter=2,
+    )
+    sscv.fit(y, X)
+    assert len(sscv.cv_results_) == 5