Repository: alan-turing-institute/sktime
Pull request #3135 (2022-07-30 13:15:07)

[ENH] Added error_score to evaluate and grid searches

#### Reference Issues/PRs
#2891 

#### What does this implement/fix? Explain your changes.
Enable better UX for grid search by avoiding exceptions. This is really important for bigger grids with multiple transformers.



Patch:
diff --git a/sktime/forecasting/compose/_pipeline.py b/sktime/forecasting/compose/_pipeline.py
index 02241e298..76112a7a0 100644
--- a/sktime/forecasting/compose/_pipeline.py
+++ b/sktime/forecasting/compose/_pipeline.py
@@ -626,7 +626,7 @@ class TransformedTargetForecaster(_Pipeline):
     >>> from sktime.transformations.series.exponent import ExponentTransformer
     >>> y = load_airline()
 
-    Example 1: string/estimator pairs
+        Example 1: string/estimator pairs
     >>> pipe = TransformedTargetForecaster(steps=[
     ...     ("imputer", Imputer(method="mean")),
     ...     ("detrender", Deseasonalizer()),
@@ -636,7 +636,7 @@ class TransformedTargetForecaster(_Pipeline):
     TransformedTargetForecaster(...)
     >>> y_pred = pipe.predict(fh=[1,2,3])
 
-    Example 2: without strings
+        Example 2: without strings
     >>> pipe = TransformedTargetForecaster([
     ...     Imputer(method="mean"),
     ...     Deseasonalizer(),
@@ -644,7 +644,7 @@ class TransformedTargetForecaster(_Pipeline):
     ...     ExponentTransformer(),
     ... ])
 
-    Example 3: using the dunder method
+        Example 3: using the dunder method
     >>> forecaster = NaiveForecaster(strategy="drift")
     >>> imputer = Imputer(method="mean")
     >>> pipe = imputer * Deseasonalizer() * forecaster * ExponentTransformer()
diff --git a/sktime/forecasting/model_evaluation/_functions.py b/sktime/forecasting/model_evaluation/_functions.py
index 277278ecc..7bfd1a1ea 100644
--- a/sktime/forecasting/model_evaluation/_functions.py
+++ b/sktime/forecasting/model_evaluation/_functions.py
@@ -7,10 +7,12 @@ __author__ = ["aiwalter", "mloning"]
 __all__ = ["evaluate"]
 
 import time
+import warnings
 
 import numpy as np
 import pandas as pd
 
+from sktime.exceptions import FitFailedWarning
 from sktime.forecasting.base import ForecastingHorizon
 from sktime.utils.validation.forecasting import (
     check_cv,
@@ -29,6 +31,7 @@ def evaluate(
     strategy="refit",
     scoring=None,
     return_data=False,
+    error_score=np.nan,
 ):
     """Evaluate forecaster using timeseries cross-validation.
 
@@ -55,6 +58,10 @@ def evaluate(
         Returns three additional columns in the DataFrame, by default False.
         The cells of the columns contain each a pd.Series for y_train,
         y_pred, y_test.
+    error_score : "raise" or numeric, default=np.nan
+        Value to assign to the score if an exception occurs in estimator fitting. If set
+        to "raise", the exception is raised. If a numeric value is given,
+        FitFailedWarning is raised.
 
     Returns
     -------
@@ -113,48 +120,74 @@ def evaluate(
 
     # Run temporal cross-validation.
     for i, (train, test) in enumerate(cv.split(y)):
+
+        # set default result values in case estimator fitting fails
+        score = error_score
+        fit_time = np.nan
+        pred_time = np.nan
+        cutoff = np.nan
+        y_pred = np.nan
+
         # split data
         y_train, y_test, X_train, X_test = _split(y, X, train, test, cv.fh)
 
         # create forecasting horizon
         fh = ForecastingHorizon(y_test.index, is_relative=False)
 
-        # fit/update
-        start_fit = time.perf_counter()
-        if i == 0 or strategy == "refit":
-            forecaster = forecaster.clone()
-            forecaster.fit(y_train, X_train, fh=fh)
-
-        else:  # if strategy in ["update", "no-update_params"]:
-            update_params = strategy == "update"
-            forecaster.update(y_train, X_train, update_params=update_params)
-        fit_time = time.perf_counter() - start_fit
-
-        pred_type = {
-            "pred_quantiles": "forecaster.predict_quantiles",
-            "pred_intervals": "forecaster.predict_interval",
-            "pred_proba": "forecaster.predict_proba",
-            None: "forecaster.predict",
-        }
-        # predict
-        start_pred = time.perf_counter()
-
-        if hasattr(scoring, "metric_args"):
-            metric_args = scoring.metric_args
-
         try:
-            scitype = scoring.get_tag("scitype:y_pred")
-        except ValueError:
-            # If no scitype exists then metric is not proba and no args needed
-            scitype = None
-            metric_args = {}
-
-        y_pred = eval(pred_type[scitype])(fh, X_test, **metric_args)
-
-        pred_time = time.perf_counter() - start_pred
-
-        # score
-        score = scoring(y_test, y_pred, y_train=y_train)
+            # fit/update
+            start_fit = time.perf_counter()
+            if i == 0 or strategy == "refit":
+                forecaster = forecaster.clone()
+                forecaster.fit(y_train, X_train, fh=fh)
+
+            else:  # if strategy in ["update", "no-update_params"]:
+                update_params = strategy == "update"
+                forecaster.update(y_train, X_train, update_params=update_params)
+            fit_time = time.perf_counter() - start_fit
+
+            pred_type = {
+                "pred_quantiles": "forecaster.predict_quantiles",
+                "pred_intervals": "forecaster.predict_interval",
+                "pred_proba": "forecaster.predict_proba",
+                None: "forecaster.predict",
+            }
+            # predict
+            start_pred = time.perf_counter()
+
+            if hasattr(scoring, "metric_args"):
+                metric_args = scoring.metric_args
+
+            try:
+                scitype = scoring.get_tag("scitype:y_pred")
+            except ValueError:
+                # If no scitype exists then metric is not proba and no args needed
+                scitype = None
+                metric_args = {}
+
+            y_pred = eval(pred_type[scitype])(fh, X_test, **metric_args)
+
+            pred_time = time.perf_counter() - start_pred
+
+            # score
+            score = scoring(y_test, y_pred, y_train=y_train)
+
+            # cutoff
+            cutoff = forecaster.cutoff
+
+        except Exception as e:
+            if error_score == "raise":
+                raise e
+            else:
+                warnings.warn(
+                    f"""
+                Fitting of forecaster failed, you can set error_score='raise' to see
+                the exception message. Fit failed for len(y_train)={len(y_train)}.
+                The score will be set to {error_score}.
+                Failed forecaster: {forecaster}.
+                """,
+                    FitFailedWarning,
+                )
 
         # save results
         results.append(
@@ -163,7 +196,7 @@ def evaluate(
                 "fit_time": fit_time,
                 "pred_time": pred_time,
                 "len_train_window": len(y_train),
-                "cutoff": forecaster.cutoff,
+                "cutoff": cutoff,
                 "y_train": y_train if return_data else np.nan,
                 "y_test": y_test if return_data else np.nan,
                 "y_pred": y_pred if return_data else np.nan,
diff --git a/sktime/forecasting/model_selection/_tune.py b/sktime/forecasting/model_selection/_tune.py
index 58e6767bd..d701063b6 100644
--- a/sktime/forecasting/model_selection/_tune.py
+++ b/sktime/forecasting/model_selection/_tune.py
@@ -42,6 +42,7 @@ class BaseGridSearch(_DelegatedForecaster):
         verbose=0,
         return_n_best_forecasters=1,
         update_behaviour="full_refit",
+        error_score=np.nan,
     ):
 
         self.forecaster = forecaster
@@ -55,6 +56,7 @@ class BaseGridSearch(_DelegatedForecaster):
         self.verbose = verbose
         self.return_n_best_forecasters = return_n_best_forecasters
         self.update_behaviour = update_behaviour
+        self.error_score = error_score
         super(BaseGridSearch, self).__init__()
         tags_to_clone = [
             "requires-fh-in-fit",
@@ -138,6 +140,7 @@ class BaseGridSearch(_DelegatedForecaster):
                 X,
                 strategy=self.strategy,
                 scoring=scoring,
+                error_score=self.error_score,
             )
 
             # Filter columns.
@@ -192,6 +195,12 @@ class BaseGridSearch(_DelegatedForecaster):
 
         # Select best parameters.
         self.best_index_ = results.loc[:, f"rank_{scoring_name}"].argmin()
+        # Raise error if all fits in evaluate failed because all score values are NaN.
+        if self.best_index_ == -1:
+            raise NotFittedError(
+                f"""All fits of forecaster failed, set error_score='raise' to see the exceptions.
+                Failed forecaster: {self.forecaster}"""
+            )
         self.best_score_ = results.loc[self.best_index_, f"mean_{scoring_name}"]
         self.best_params_ = results.loc[self.best_index_, "params"]
         self.best_forecaster_ = self.forecaster.clone().set_params(**self.best_params_)
@@ -315,6 +324,10 @@ class ForecastingGridSearchCV(BaseGridSearch):
     backend: str, optional (default="loky")
         Specify the parallelisation backend implementation in joblib, where
         "loky" is used by default.
+    error_score : "raise" or numeric, default=np.nan
+        Value to assign to the score if an exception occurs in estimator fitting. If set
+        to "raise", the exception is raised. If a numeric value is given,
+        FitFailedWarning is raised.
 
     Attributes
     ----------
@@ -420,6 +433,7 @@ class ForecastingGridSearchCV(BaseGridSearch):
         pre_dispatch="2*n_jobs",
         backend="loky",
         update_behaviour="full_refit",
+        error_score=np.nan,
     ):
         super(ForecastingGridSearchCV, self).__init__(
             forecaster=forecaster,
@@ -433,6 +447,7 @@ class ForecastingGridSearchCV(BaseGridSearch):
             pre_dispatch=pre_dispatch,
             backend=backend,
             update_behaviour=update_behaviour,
+            error_score=error_score,
         )
         self.param_grid = param_grid
 
@@ -566,6 +581,10 @@ class ForecastingRandomizedSearchCV(BaseGridSearch):
     backend: str, optional (default="loky")
         Specify the parallelisation backend implementation in joblib, where
         "loky" is used by default.
+    error_score : "raise" or numeric, default=np.nan
+        Value to assign to the score if an exception occurs in estimator fitting. If set
+        to "raise", the exception is raised. If a numeric value is given,
+        FitFailedWarning is raised.
 
     Attributes
     ----------
@@ -603,6 +622,7 @@ class ForecastingRandomizedSearchCV(BaseGridSearch):
         pre_dispatch="2*n_jobs",
         backend="loky",
         update_behaviour="full_refit",
+        error_score=np.nan,
     ):
         super(ForecastingRandomizedSearchCV, self).__init__(
             forecaster=forecaster,
@@ -616,6 +636,7 @@ class ForecastingRandomizedSearchCV(BaseGridSearch):
             pre_dispatch=pre_dispatch,
             backend=backend,
             update_behaviour=update_behaviour,
+            error_score=error_score,
         )
         self.param_distributions = param_distributions
         self.n_iter = n_iter


Test patch:
diff --git a/sktime/forecasting/model_evaluation/tests/test_evaluate.py b/sktime/forecasting/model_evaluation/tests/test_evaluate.py
index 8e009616f..3a2995db7 100644
--- a/sktime/forecasting/model_evaluation/tests/test_evaluate.py
+++ b/sktime/forecasting/model_evaluation/tests/test_evaluate.py
@@ -18,8 +18,10 @@ import numpy as np
 import pandas as pd
 import pytest
 
-from sktime.datasets import load_longley
+from sktime.datasets import load_airline, load_longley
+from sktime.exceptions import FitFailedWarning
 from sktime.forecasting.arima import ARIMA
+from sktime.forecasting.exp_smoothing import ExponentialSmoothing
 from sktime.forecasting.model_evaluation import evaluate
 from sktime.forecasting.model_selection import (
     ExpandingWindowSplitter,
@@ -155,3 +157,42 @@ def test_evaluate_no_exog_against_with_exog():
 
     scoring_name = f"test_{scoring.name}"
     assert np.all(out_exog[scoring_name] != out_no_exog[scoring_name])
+
+
+@pytest.mark.parametrize("error_score", [np.nan, "raise", 1000])
+@pytest.mark.parametrize("return_data", [True, False])
+@pytest.mark.parametrize("strategy", ["refit", "update"])
+def test_evaluate_error_score(error_score, return_data, strategy):
+    """Test evaluate to raise warnings and exceptions according to error_score value."""
+    forecaster = ExponentialSmoothing(sp=12)
+    y = load_airline()
+    # add NaN to make ExponentialSmoothing fail
+    y.iloc[1] = np.nan
+    fh = [1, 2, 3]
+    cv = ExpandingWindowSplitter(
+        start_with_window=True, step_length=48, initial_window=12, fh=fh
+    )
+    if error_score in [np.nan, 1000]:
+        with pytest.warns(FitFailedWarning):
+            results = evaluate(
+                forecaster=forecaster,
+                y=y,
+                cv=cv,
+                return_data=return_data,
+                error_score=error_score,
+                strategy=strategy,
+            )
+        if isinstance(error_score, type(np.nan)):
+            assert results["test_MeanAbsolutePercentageError"].isna().sum() > 0
+        if error_score == 1000:
+            assert results["test_MeanAbsolutePercentageError"].max() == 1000
+    if error_score == "raise":
+        with pytest.raises(Exception):
+            evaluate(
+                forecaster=forecaster,
+                y=y,
+                cv=cv,
+                return_data=return_data,
+                error_score=error_score,
+                strategy=strategy,
+            )
diff --git a/sktime/forecasting/model_selection/tests/test_tune.py b/sktime/forecasting/model_selection/tests/test_tune.py
index 237375d7a..bc30a89e1 100644
--- a/sktime/forecasting/model_selection/tests/test_tune.py
+++ b/sktime/forecasting/model_selection/tests/test_tune.py
@@ -76,6 +76,7 @@ CVs = [
     *[SingleWindowSplitter(fh=fh) for fh in TEST_OOS_FHS],
     SlidingWindowSplitter(fh=1, initial_window=15),
 ]
+ERROR_SCORES = [np.nan, "raise", 1000]
 
 
 @pytest.mark.parametrize(
@@ -83,11 +84,16 @@ CVs = [
 )
 @pytest.mark.parametrize("scoring", TEST_METRICS)
 @pytest.mark.parametrize("cv", CVs)
-def test_gscv(forecaster, param_grid, cv, scoring):
+@pytest.mark.parametrize("error_score", ERROR_SCORES)
+def test_gscv(forecaster, param_grid, cv, scoring, error_score):
     """Test ForecastingGridSearchCV."""
     y, X = load_longley()
     gscv = ForecastingGridSearchCV(
-        forecaster, param_grid=param_grid, cv=cv, scoring=scoring
+        forecaster,
+        param_grid=param_grid,
+        cv=cv,
+        scoring=scoring,
+        error_score=error_score,
     )
     gscv.fit(y, X)
 
@@ -99,10 +105,11 @@ def test_gscv(forecaster, param_grid, cv, scoring):
     "forecaster, param_grid", [(NAIVE, NAIVE_GRID), (PIPE, PIPE_GRID)]
 )
 @pytest.mark.parametrize("scoring", TEST_METRICS)
+@pytest.mark.parametrize("error_score", ERROR_SCORES)
 @pytest.mark.parametrize("cv", CVs)
 @pytest.mark.parametrize("n_iter", TEST_N_ITERS)
 @pytest.mark.parametrize("random_state", TEST_RANDOM_SEEDS)
-def test_rscv(forecaster, param_grid, cv, scoring, n_iter, random_state):
+def test_rscv(forecaster, param_grid, cv, scoring, error_score, n_iter, random_state):
     """Test ForecastingRandomizedSearchCV.
 
     Tests that ForecastingRandomizedSearchCV successfully searches the
@@ -114,6 +121,7 @@ def test_rscv(forecaster, param_grid, cv, scoring, n_iter, random_state):
         param_distributions=param_grid,
         cv=cv,
         scoring=scoring,
+        error_score=error_score,
         n_iter=n_iter,
         random_state=random_state,
     )
diff --git a/sktime/utils/_testing/deep_equals.py b/sktime/utils/_testing/deep_equals.py
index d8834c2a0..8a7b7d4cb 100644
--- a/sktime/utils/_testing/deep_equals.py
+++ b/sktime/utils/_testing/deep_equals.py
@@ -112,6 +112,10 @@ def deep_equals(x, y, return_msg=False):
         return ret(*_tuple_equals(x, y, return_msg=True))
     elif isinstance(x, dict):
         return ret(*_dict_equals(x, y, return_msg=True))
+    elif isinstance(x, type(np.nan)):
+        return ret(
+            isinstance(y, type(np.nan)), f"type(x)={type(x)} != type(y)={type(y)}"
+        )
     elif type(x).__name__ == "ForecastingHorizon":
         return ret(*_fh_equals(x, y, return_msg=True))
     elif x != y: