Repository: sktime/sktime

Issue:
[BUG] `BaseForecaster.update_predict` broken upon second call, `detached_cutoff` does not reset component cutoffs
`update_predict` is broken for all composite forecasters in the following way:

If it is run once for a composite with a component, the `cutoff` of the composite is reset, but not that of the component. This leads to unexpected behaviour and non-compliant predictions when `update_predict` is called a second time.

This is due to a fundamental bug in the design of the `detached_cutoff` that is used internally.

An example is the test failure in https://github.com/alan-turing-institute/sktime/pull/2248, in combination with #2555.

Another minimal example of this behaviour is this, in #2248:

```python
from sktime.utils._testing.series import _make_series
from sktime.forecasting.compose import BaggingForecaster
from sktime.forecasting.model_selection import SlidingWindowSplitter

X = _make_series()
X1 = X[:24]
X2 = X[24:36]
f = BaggingForecaster.create_test_instance()
f.fit(X1, fh=[1, 2])

cv = SlidingWindowSplitter(start_with_window=False)
f.update_predict(X2, cv=cv)
f.update_predict(X2, cv=cv)
```

The expected behaviour is that returns of both `update_predict` are identical, and that the `cutoff` of the internal estimator does not change.


Discussion:

Resolving patch:
diff --git a/sktime/forecasting/base/_base.py b/sktime/forecasting/base/_base.py
--- a/sktime/forecasting/base/_base.py
+++ b/sktime/forecasting/base/_base.py
@@ -37,7 +37,7 @@ class name: BaseForecaster
 
 __all__ = ["BaseForecaster"]
 
-from contextlib import contextmanager
+from copy import deepcopy
 from warnings import warn
 
 import numpy as np
@@ -771,6 +771,7 @@ def update_predict(
         cv=None,
         X=None,
         update_params=True,
+        reset_forecaster=True,
     ):
         """Make predictions and update model iteratively over the test set.
 
@@ -783,12 +784,14 @@ def update_predict(
             self.cutoff, self._is_fitted
             If update_params=True, model attributes ending in "_".
 
-        Writes to self:
+        Writes to self, if reset_forecaster=False:
             Update self._y and self._X with `y` and `X`, by appending rows.
             Updates self.cutoff and self._cutoff to last index seen in `y`.
             If update_params=True,
                 updates fitted model attributes ending in "_".
 
+        Does not update state if reset_forecaster=True.
+
         Parameters
         ----------
         y : time series in sktime compatible data container format
@@ -818,6 +821,13 @@ def update_predict(
                 X.index must contain y.index and fh.index both
             there are no restrictions on number of columns (unlike for y)
         update_params : bool, optional (default=True)
+            whether model parameters should be updated in each update step
+        reset_forecaster : bool, optional (default=True)
+            if True, will not change the state of the forecaster,
+                i.e., update/predict sequence is run with a copy,
+                and cutoff, model parameters, data memory of self do not change
+            if False, will update self when the update/predict sequence is run
+                as if update/predict were called directly
 
         Returns
         -------
@@ -838,6 +848,7 @@ def update_predict(
             cv=cv,
             X=X_inner,
             update_params=update_params,
+            reset_forecaster=reset_forecaster,
         )
 
     def update_predict_single(
@@ -1405,24 +1416,6 @@ def _set_cutoff_from_y(self, y):
         cutoff_idx = get_cutoff(y, self.cutoff)
         self._cutoff = cutoff_idx
 
-    @contextmanager
-    def _detached_cutoff(self):
-        """Detached cutoff mode.
-
-        When in detached cutoff mode, the cutoff can be updated but will
-        be reset to the initial value after leaving the detached cutoff mode.
-
-        This is useful during rolling-cutoff forecasts when the cutoff needs
-        to be repeatedly reset, but afterwards should be restored to the
-        original value.
-        """
-        cutoff = self.cutoff  # keep initial cutoff
-        try:
-            yield
-        finally:
-            # re-set cutoff to initial value
-            self._set_cutoff(cutoff)
-
     @property
     def fh(self):
         """Forecasting horizon that was passed."""
@@ -2007,15 +2000,47 @@ def _predict_proba(self, fh, X, marginal=True):
 
         return pred_dist
 
-    def _predict_moving_cutoff(self, y, cv, X=None, update_params=True):
+    def _predict_moving_cutoff(
+        self, y, cv, X=None, update_params=True, reset_forecaster=True
+    ):
         """Make single-step or multi-step moving cutoff predictions.
 
         Parameters
         ----------
-        y : pd.Series
-        cv : temporal cross-validation generator
-        X : pd.DataFrame
-        update_params : bool
+        y : time series in sktime compatible data container format
+                Time series to which to fit the forecaster in the update.
+            y can be in one of the following formats, must be same scitype as in fit:
+            Series scitype: pd.Series, pd.DataFrame, or np.ndarray (1D or 2D)
+                for vanilla forecasting, one time series
+            Panel scitype: pd.DataFrame with 2-level row MultiIndex,
+                3D np.ndarray, list of Series pd.DataFrame, or nested pd.DataFrame
+                for global or panel forecasting
+            Hierarchical scitype: pd.DataFrame with 3 or more level row MultiIndex
+                for hierarchical forecasting
+            Number of columns admissible depend on the "scitype:y" tag:
+                if self.get_tag("scitype:y")=="univariate":
+                    y must have a single column/variable
+                if self.get_tag("scitype:y")=="multivariate":
+                    y must have 2 or more columns
+                if self.get_tag("scitype:y")=="both": no restrictions on columns apply
+            For further details:
+                on usage, see forecasting tutorial examples/01_forecasting.ipynb
+                on specification of formats, examples/AA_datatypes_and_datasets.ipynb
+        cv : temporal cross-validation generator, optional (default=None)
+        X : time series in sktime compatible format, optional (default=None)
+                Exogeneous time series for updating and forecasting
+            Should be of same scitype (Series, Panel, or Hierarchical) as y
+            if self.get_tag("X-y-must-have-same-index"),
+                X.index must contain y.index and fh.index both
+            there are no restrictions on number of columns (unlike for y)
+        update_params : bool, optional (default=True)
+            whether model parameters should be updated in each update step
+        reset_forecaster : bool, optional (default=True)
+            if True, will not change the state of the forecaster,
+                i.e., update/predict sequence is run with a copy,
+                and cutoff, model parameters, data memory of self do not change
+            if False, will update self when the update/predict sequence is run
+                as if update/predict were called directly
 
         Returns
         -------
@@ -2025,32 +2050,37 @@ def _predict_moving_cutoff(self, y, cv, X=None, update_params=True):
         y_preds = []
         cutoffs = []
 
-        # enter into a detached cutoff mode
-        with self._detached_cutoff():
-            # set cutoff to time point before data
-            self._set_cutoff(_shift(y.index[0], by=-1))
-            # iterate over data
-            for new_window, _ in cv.split(y):
-                y_new = y.iloc[new_window]
-
-                # we use `update_predict_single` here
-                #  this updates the forecasting horizon
-                y_pred = self.update_predict_single(
-                    y=y_new,
-                    fh=fh,
-                    X=X,
-                    update_params=update_params,
+        # enter into a detached cutoff mode, if reset_forecaster is True
+        if reset_forecaster:
+            self_copy = deepcopy(self)
+        # otherwise just work with a reference to self
+        else:
+            self_copy = self
+
+        # set cutoff to time point before data
+        self_copy._set_cutoff(_shift(y.index[0], by=-1))
+        # iterate over data
+        for new_window, _ in cv.split(y):
+            y_new = y.iloc[new_window]
+
+            # we use `update_predict_single` here
+            #  this updates the forecasting horizon
+            y_pred = self_copy.update_predict_single(
+                y=y_new,
+                fh=fh,
+                X=X,
+                update_params=update_params,
+            )
+            y_preds.append(y_pred)
+            cutoffs.append(self_copy.cutoff)
+
+            for i in range(len(y_preds)):
+                y_preds[i] = convert_to(
+                    y_preds[i],
+                    self._y_mtype_last_seen,
+                    store=self._converter_store_y,
+                    store_behaviour="freeze",
                 )
-                y_preds.append(y_pred)
-                cutoffs.append(self.cutoff)
-
-                for i in range(len(y_preds)):
-                    y_preds[i] = convert_to(
-                        y_preds[i],
-                        self._y_mtype_last_seen,
-                        store=self._converter_store_y,
-                        store_behaviour="freeze",
-                    )
         return _format_moving_cutoff_predictions(y_preds, cutoffs)
 
     # TODO: remove in v0.11.0


Test patch:
diff --git a/sktime/forecasting/online_learning/tests/test_online_learning.py b/sktime/forecasting/online_learning/tests/test_online_learning.py
--- a/sktime/forecasting/online_learning/tests/test_online_learning.py
+++ b/sktime/forecasting/online_learning/tests/test_online_learning.py
@@ -6,19 +6,20 @@
 __author__ = ["magittan"]
 
 import numpy as np
+from sklearn.metrics import mean_squared_error
+
 from sktime.datasets import load_airline
-from sktime.forecasting.model_selection import temporal_train_test_split
+from sktime.forecasting.exp_smoothing import ExponentialSmoothing
+from sktime.forecasting.model_selection import (
+    SlidingWindowSplitter,
+    temporal_train_test_split,
+)
+from sktime.forecasting.naive import NaiveForecaster
+from sktime.forecasting.online_learning._online_ensemble import OnlineEnsembleForecaster
 from sktime.forecasting.online_learning._prediction_weighted_ensembler import (
-    NormalHedgeEnsemble,
     NNLSEnsemble,
+    NormalHedgeEnsemble,
 )
-from sktime.forecasting.online_learning._online_ensemble import (
-    OnlineEnsembleForecaster,
-)
-from sktime.forecasting.model_selection import SlidingWindowSplitter
-from sktime.forecasting.exp_smoothing import ExponentialSmoothing
-from sktime.forecasting.naive import NaiveForecaster
-from sklearn.metrics import mean_squared_error
 
 cv = SlidingWindowSplitter(start_with_window=True, window_length=1, fh=1)
 
@@ -69,7 +70,7 @@ def test_weights_for_airline_normal_hedge():
     )
 
     forecaster.fit(y_train)
-    forecaster.update_predict(y=y_test, cv=cv)
+    forecaster.update_predict(y=y_test, cv=cv, reset_forecaster=False)
 
     expected = np.array([0.17077154, 0.48156709, 0.34766137])
     np.testing.assert_allclose(forecaster.weights, expected, atol=1e-8)
@@ -92,7 +93,7 @@ def test_weights_for_airline_nnls():
     )
 
     forecaster.fit(y_train)
-    forecaster.update_predict(y=y_test, cv=cv)
+    forecaster.update_predict(y=y_test, cv=cv, reset_forecaster=False)
 
     expected = np.array([0.04720766, 0, 1.03410876])
     np.testing.assert_allclose(forecaster.weights, expected, atol=1e-8)