Repository: sktime/sktime

Issue:
Random Grid Search for Forecasting
**Is your feature request related to a problem? Please describe.**
I am wondering if there is a Random Grid Search equivalent of [`ForecastingGridSearchCV`](https://www.sktime.org/en/latest/api_reference/modules/auto_generated/sktime.forecasting.model_selection.ForecastingGridSearchCV.html#forecastinggridsearchcv) similar to how we have [`RandomizedSearchCV`](https://scikit-learn.org/stable/modules/generated/sklearn.model_selection.RandomizedSearchCV.html) equivalent of [`GridSearchCV`](https://scikit-learn.org/stable/modules/generated/sklearn.model_selection.GridSearchCV.html) in scikit-learn?

**Describe the solution you'd like**
Would like to have a random grid search equivalent of the fixed grid search option (similar to what is present in scikit-learn).


**Describe alternatives you've considered**
None so far




Discussion:
I'd like to have a go at this, please!
@pabworks great - would really appreciate a PR, let us know if you need any help! 

Resolving patch:
diff --git a/sktime/forecasting/all/__init__.py b/sktime/forecasting/all/__init__.py
--- a/sktime/forecasting/all/__init__.py
+++ b/sktime/forecasting/all/__init__.py
@@ -10,6 +10,7 @@
     "load_shampoo_sales",
     "CutoffSplitter",
     "ForecastingGridSearchCV",
+    "ForecastingRandomizedSearchCV",
     "SlidingWindowSplitter",
     "SingleWindowSplitter",
     "temporal_train_test_split",
@@ -51,7 +52,10 @@
 from sktime.forecasting.compose import TransformedTargetForecaster
 from sktime.forecasting.exp_smoothing import ExponentialSmoothing
 from sktime.forecasting.model_selection import CutoffSplitter
-from sktime.forecasting.model_selection import ForecastingGridSearchCV
+from sktime.forecasting.model_selection import (
+    ForecastingGridSearchCV,
+    ForecastingRandomizedSearchCV,
+)
 from sktime.forecasting.model_selection import SingleWindowSplitter
 from sktime.forecasting.model_selection import SlidingWindowSplitter
 from sktime.forecasting.model_selection import temporal_train_test_split
diff --git a/sktime/forecasting/model_selection/__init__.py b/sktime/forecasting/model_selection/__init__.py
--- a/sktime/forecasting/model_selection/__init__.py
+++ b/sktime/forecasting/model_selection/__init__.py
@@ -10,6 +10,7 @@
     "temporal_train_test_split",
     "ExpandingWindowSplitter",
     "ForecastingGridSearchCV",
+    "ForecastingRandomizedSearchCV",
 ]
 
 from sktime.forecasting.model_selection._split import ExpandingWindowSplitter
@@ -18,3 +19,4 @@
 from sktime.forecasting.model_selection._split import SlidingWindowSplitter
 from sktime.forecasting.model_selection._split import temporal_train_test_split
 from sktime.forecasting.model_selection._tune import ForecastingGridSearchCV
+from sktime.forecasting.model_selection._tune import ForecastingRandomizedSearchCV
diff --git a/sktime/forecasting/model_selection/_tune.py b/sktime/forecasting/model_selection/_tune.py
--- a/sktime/forecasting/model_selection/_tune.py
+++ b/sktime/forecasting/model_selection/_tune.py
@@ -3,7 +3,7 @@
 # copyright: sktime developers, BSD-3-Clause License (see LICENSE file)
 
 __author__ = ["Markus Löning"]
-__all__ = ["ForecastingGridSearchCV"]
+__all__ = ["ForecastingGridSearchCV", "ForecastingRandomizedSearchCV"]
 
 import numbers
 import time
@@ -17,7 +17,7 @@
 import pandas as pd
 from scipy.stats import rankdata
 from sklearn.base import clone
-from sklearn.model_selection import ParameterGrid
+from sklearn.model_selection import ParameterGrid, ParameterSampler
 from sklearn.model_selection import check_cv
 from sklearn.model_selection._search import _check_param_grid
 from sklearn.model_selection._validation import _aggregate_score_dicts
@@ -576,3 +576,113 @@ def __init__(
     def _run_search(self, evaluate_candidates):
         """Search all candidates in param_grid"""
         evaluate_candidates(ParameterGrid(self.param_grid))
+
+
+class ForecastingRandomizedSearchCV(BaseGridSearch):
+    """
+    Performs randomized-search cross-validation to find optimal model parameters.
+    The forecaster is fit on the initial window and then temporal
+    cross-validation is used to find the optimal parameter
+
+    Randomized cross-validation is performed based on a cross-validation
+    iterator encoding the cross-validation scheme, the parameter distributions to
+    search over, and (optionally) the evaluation metric for comparing model
+    performance. As in scikit-learn, tuning works through the common
+    hyper-parameter interface which allows to repeatedly fit and evaluate
+    the same forecaster with different hyper-parameters.
+
+    Parameters
+    ----------
+    forecaster : estimator object
+        The estimator should implement the sktime or scikit-learn estimator
+        interface. Either the estimator must contain a "score" function,
+        or a scoring function must be passed.
+    cv : cross-validation generator or an iterable
+        e.g. SlidingWindowSplitter()
+    param_distributions : dict or list of dicts
+        Dictionary with parameters names (`str`) as keys and distributions
+        or lists of parameters to try. Distributions must provide a ``rvs``
+        method for sampling (such as those from scipy.stats.distributions).
+        If a list is given, it is sampled uniformly.
+        If a list of dicts is given, first a dict is sampled uniformly, and
+        then a parameter is sampled using that dict as above.
+    n_iter : int, default=10
+        Number of parameter settings that are sampled. n_iter trades
+        off runtime vs quality of the solution.
+    scoring: function, optional (default=None)
+        Function to score models for evaluation of optimal parameters
+    n_jobs: int, optional (default=None)
+        Number of jobs to run in parallel.
+        None means 1 unless in a joblib.parallel_backend context.
+        -1 means using all processors.
+    refit: bool, optional (default=True)
+        Refit the forecaster with the best parameters on all the data
+    verbose: int, optional (default=0)
+    random_state : int, RandomState instance or None, default=None
+        Pseudo random number generator state used for random uniform sampling
+        from lists of possible values instead of scipy.stats distributions.
+        Pass an int for reproducible output across multiple
+        function calls.
+    pre_dispatch: str, optional (default='2*n_jobs')
+    error_score: numeric value or the str 'raise', optional (default=np.nan)
+        The test score returned when a forecaster fails to be fitted.
+    return_train_score: bool, optional (default=False)
+
+    Attributes
+    ----------
+    best_index_ : int
+    best_score_: float
+        Score of the best model
+    best_params_ : dict
+        Best parameter values across the parameter grid
+    best_forecaster_ : estimator
+        Fitted estimator with the best parameters
+    cv_results_ : dict
+        Results from grid search cross validation
+    n_splits_: int
+        Number of splits in the data for cross validation}
+    refit_time_ : float
+        Time (seconds) to refit the best forecaster
+    scorer_ : function
+        Function used to score model
+    """
+
+    _required_parameters = ["forecaster", "cv", "param_distributions"]
+
+    def __init__(
+        self,
+        forecaster,
+        cv,
+        param_distributions,
+        n_iter=10,
+        scoring=None,
+        n_jobs=None,
+        refit=True,
+        verbose=0,
+        random_state=None,
+        pre_dispatch="2*n_jobs",
+        error_score=np.nan,
+        return_train_score=False,
+    ):
+        super(ForecastingRandomizedSearchCV, self).__init__(
+            forecaster=forecaster,
+            scoring=scoring,
+            n_jobs=n_jobs,
+            refit=refit,
+            cv=cv,
+            verbose=verbose,
+            pre_dispatch=pre_dispatch,
+            error_score=error_score,
+            return_train_score=return_train_score,
+        )
+        self.param_distributions = param_distributions
+        self.n_iter = n_iter
+        self.random_state = random_state
+
+    def _run_search(self, evaluate_candidates):
+        """Search n_iter candidates from param_distributions"""
+        evaluate_candidates(
+            ParameterSampler(
+                self.param_distributions, self.n_iter, random_state=self.random_state
+            )
+        )


Test patch:
diff --git a/sktime/forecasting/model_selection/tests/test_tune.py b/sktime/forecasting/model_selection/tests/test_tune.py
--- a/sktime/forecasting/model_selection/tests/test_tune.py
+++ b/sktime/forecasting/model_selection/tests/test_tune.py
@@ -3,25 +3,28 @@
 # copyright: sktime developers, BSD-3-Clause License (see LICENSE file)
 
 __author__ = ["Markus Löning"]
-__all__ = ["test_gscv_fit"]
+__all__ = ["test_gscv_fit", "test_rscv_fit"]
 
 import numpy as np
 import pytest
 from sklearn.base import clone
 from sklearn.linear_model import LinearRegression
 from sklearn.metrics import mean_squared_error
-from sklearn.model_selection import ParameterGrid
+from sklearn.model_selection import ParameterGrid, ParameterSampler
 
 from sktime.datasets import load_airline
 from sktime.forecasting.compose import ReducedRegressionForecaster
 from sktime.forecasting.compose import TransformedTargetForecaster
 from sktime.forecasting.model_selection import ForecastingGridSearchCV
+from sktime.forecasting.model_selection import ForecastingRandomizedSearchCV
 from sktime.forecasting.model_selection import SingleWindowSplitter
 from sktime.forecasting.model_selection import SlidingWindowSplitter
 from sktime.forecasting.naive import NaiveForecaster
 from sktime.forecasting.tests._config import TEST_OOS_FHS
 from sktime.forecasting.tests._config import TEST_STEP_LENGTHS
 from sktime.forecasting.tests._config import TEST_WINDOW_LENGTHS
+from sktime.forecasting.tests._config import TEST_RANDOM_SEEDS
+from sktime.forecasting.tests._config import TEST_N_ITERS
 from sktime.forecasting.trend import PolynomialTrendForecaster
 from sktime.performance_metrics.forecasting import make_forecasting_scorer
 from sktime.performance_metrics.forecasting import sMAPE
@@ -102,3 +105,82 @@ def test_gscv_fit(forecaster, param_dict, cv, scoring):
         for key, value in gscv.best_forecaster_.get_params().items()
         if key in gscv.best_params_.keys()
     } == gscv.best_params_
+
+
+@pytest.mark.parametrize(
+    "forecaster, param_dict",
+    [
+        (NaiveForecaster(strategy="mean"), {"window_length": TEST_WINDOW_LENGTHS}),
+        # atomic estimator
+        (
+            TransformedTargetForecaster(
+                [  # composite estimator
+                    ("t", Detrender(PolynomialTrendForecaster())),
+                    ("f", ReducedRegressionForecaster(LinearRegression())),
+                ]
+            ),
+            {
+                "f__window_length": TEST_WINDOW_LENGTHS,
+                "f__step_length": TEST_STEP_LENGTHS,
+            },
+        ),  # multiple params
+    ],
+)
+@pytest.mark.parametrize(
+    "scoring",
+    [sMAPE(), make_forecasting_scorer(mean_squared_error, greater_is_better=False)],
+)
+@pytest.mark.parametrize(
+    "cv",
+    [
+        *[SingleWindowSplitter(fh=fh) for fh in TEST_OOS_FHS],
+        # single split with multi-step fh
+        SlidingWindowSplitter(fh=1, initial_window=50)
+        # multiple splits with single-step fh
+    ],
+)
+@pytest.mark.parametrize(
+    "n_iter",
+    TEST_N_ITERS,
+)
+@pytest.mark.parametrize(
+    "random_state",
+    TEST_RANDOM_SEEDS,
+)
+def test_rscv_fit(forecaster, param_dict, cv, scoring, n_iter, random_state):
+    """Tests that ForecastingRandomizedSearchCV successfully searches the
+    parameter distributions to identify the best parameter set
+    """
+    # samples uniformly from param dict values
+    param_distributions = ParameterSampler(
+        param_dict, n_iter, random_state=random_state
+    )
+
+    y = load_airline()
+    rscv = ForecastingRandomizedSearchCV(
+        forecaster,
+        param_distributions=param_dict,
+        cv=cv,
+        scoring=scoring,
+        n_iter=n_iter,
+        random_state=random_state,
+    )
+    rscv.fit(y)
+
+    # check scores
+    rscv_scores = rscv.cv_results_[f"mean_test_{scoring.name}"]
+    # convert ParameterSampler to list to ensure consistent # of scores
+    expected_scores = compute_expected_gscv_scores(
+        forecaster, cv, list(param_distributions), y, scoring
+    )
+    np.testing.assert_array_equal(rscv_scores, expected_scores)
+
+    # check best parameters
+    assert rscv.best_params_ == list(param_distributions)[rscv_scores.argmin()]
+
+    # check best forecaster is the one with best parameters
+    assert {
+        key: value
+        for key, value in rscv.best_forecaster_.get_params().items()
+        if key in rscv.best_params_.keys()
+    } == rscv.best_params_
diff --git a/sktime/forecasting/tests/_config.py b/sktime/forecasting/tests/_config.py
--- a/sktime/forecasting/tests/_config.py
+++ b/sktime/forecasting/tests/_config.py
@@ -13,6 +13,8 @@
     "TEST_WINDOW_LENGTHS",
     "VALID_INDEX_FH_COMBINATIONS",
     "INDEX_TYPE_LOOKUP",
+    "TEST_RANDOM_SEEDS",
+    "TEST_N_ITERS",
 ]
 
 import numpy as np
@@ -36,6 +38,8 @@
 TEST_YS = [
     _make_series(all_positive=True),
 ]
+TEST_RANDOM_SEEDS = [1, 42]
+TEST_N_ITERS = [1, 4]
 
 # We currently support the following combinations of index and forecasting horizon types
 VALID_INDEX_FH_COMBINATIONS = [
diff --git a/sktime/tests/_config.py b/sktime/tests/_config.py
--- a/sktime/tests/_config.py
+++ b/sktime/tests/_config.py
@@ -38,6 +38,7 @@
 from sktime.forecasting.exp_smoothing import ExponentialSmoothing
 from sktime.forecasting.hcrystalball import HCrystalBallForecaster
 from sktime.forecasting.model_selection import ForecastingGridSearchCV
+from sktime.forecasting.model_selection import ForecastingRandomizedSearchCV
 from sktime.forecasting.model_selection import SingleWindowSplitter
 from sktime.forecasting.naive import NaiveForecaster
 from sktime.forecasting.online_learning import OnlineEnsembleForecaster
@@ -144,6 +145,12 @@
         "param_grid": {"window_length": [2, 5]},
         "scoring": sMAPE(),
     },
+    ForecastingRandomizedSearchCV: {
+        "forecaster": NaiveForecaster(strategy="mean"),
+        "cv": SingleWindowSplitter(fh=1),
+        "param_distributions": {"window_length": [2, 5]},
+        "scoring": sMAPE(),
+    },
     TabularToSeriesAdaptor: {"transformer": StandardScaler()},
     ColumnEnsembleClassifier: {
         "estimators": [