Repository: autogluon/autogluon
Pull request: [timeseries] Add cutoff support in evaluate and leaderboard
*Issue #, if available:* fixes #4356 

*Description of changes:* This PR adds support for `cutoff` in `TimeSeriesPredictor.evaluate` and `TimeSeriesPredictor.leaderboard`. 

TODOs:

- [x] Add support in `leaderboard`
- [x] Add tests
- [x] Update docstrings


By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice.


Problem statement:
[timeseries] Allow choosing custom cutoff in `evaluate` and `leaderboard`
## Description
Currently, methods like `evaluate()` and `leaderboard()` always use the last `prediction_length` time steps of each time series as the test. If users want to perform some more sophisticated evaluation, they need to define complex splitting logic outside of the predictor. We can simplify this process a lot by adding a `cutoff` parameter to these methods that defaults to `-1 * prediction_length` (current behavior).

Hints from issue discussion:

Commit message: Only modify predictor

Patch to review:
diff --git a/timeseries/src/autogluon/timeseries/predictor.py b/timeseries/src/autogluon/timeseries/predictor.py
index 7e1404f9980..b2ddbccadfa 100644
--- a/timeseries/src/autogluon/timeseries/predictor.py
+++ b/timeseries/src/autogluon/timeseries/predictor.py
@@ -305,15 +305,32 @@ def _check_and_prepare_data_frame(
                 df = df.convert_frequency(freq=self.freq)
         return df
 
-    def _check_data_for_evaluation(self, data: TimeSeriesDataFrame, name: str = "data") -> None:
-        """Make sure that provided evaluation data includes both historical and future time series values."""
-        if data.num_timesteps_per_item().min() <= self.prediction_length:
+    def _check_and_prepare_data_frame_for_evaluation(
+        self, data: TimeSeriesDataFrame, cutoff: Optional[int] = None, name: str = "data"
+    ) -> TimeSeriesDataFrame:
+        """
+        Make sure that provided evaluation data includes both historical and future time series values.
+        Slices the dataframe based on cutoff, if needed.
+        """
+        cutoff = -1 * self.prediction_length if cutoff is None else cutoff
+        if not (isinstance(cutoff, int) and cutoff <= -self.prediction_length):
+            raise ValueError(f"`cutoff` should be a negative integer <= -prediction_length, got: {cutoff=}")
+
+        expected_length = -cutoff
+
+        if data.num_timesteps_per_item().min() <= expected_length:
+            var_name = "-cutoff" if expected_length > self.prediction_length else "prediction_length"
             raise ValueError(
-                f"Cannot reserve last prediction_length={self.prediction_length} time steps for evaluation in some "
+                f"Cannot reserve last {expected_length} time steps for evaluation in some "
                 f"time series in {name}. Please make sure that {name} includes both historical and future data, and that"
-                f"all time series have length > prediction_length (at least {self.prediction_length + 1})"
+                f"all time series have length > {var_name} (at least {expected_length + 1})"
             )
 
+        if cutoff < -self.prediction_length:
+            data = data.slice_by_timestep(None, cutoff + self.prediction_length)
+
+        return data
+
     def _get_dataset_stats(self, data: TimeSeriesDataFrame) -> str:
         ts_lengths = data.num_timesteps_per_item()
         median_length = ts_lengths.median()
@@ -701,7 +718,7 @@ def fit(
 
         if tuning_data is not None:
             tuning_data = self._check_and_prepare_data_frame(tuning_data, name="tuning_data")
-            self._check_data_for_evaluation(tuning_data, name="tuning_data")
+            tuning_data = self._check_and_prepare_data_frame_for_evaluation(tuning_data, name="tuning_data")
             logger.info(f"Provided tuning_data has {self._get_dataset_stats(tuning_data)}")
             # TODO: Use num_val_windows to perform multi-window backtests on tuning_data
             if num_val_windows > 0:
@@ -844,6 +861,7 @@ def predict(
     def evaluate(
         self,
         data: Union[TimeSeriesDataFrame, pd.DataFrame, Path, str],
+        cutoff: Optional[int] = None,
         model: Optional[str] = None,
         metrics: Optional[Union[str, TimeSeriesScorer, List[Union[str, TimeSeriesScorer]]]] = None,
         display: bool = False,
@@ -863,17 +881,24 @@ def evaluate(
         Parameters
         ----------
         data : Union[TimeSeriesDataFrame, pd.DataFrame, Path, str]
-            The data to evaluate the best model on. The last ``prediction_length`` time steps of each time series in
-            ``data`` will be held out for prediction and forecast accuracy will be calculated on these time steps.
+            The data to evaluate the best model on. If a ``cutoff`` is not provided, the last ``prediction_length``
+            time steps of each time series in ``data`` will be held out for prediction and forecast accuracy will
+            be calculated on these time steps. When a ``cutoff`` is provided, the ``-cutoff``-th to the
+            ``-cutoff + prediction_length``-th time steps of each time series are used for evaluation.
 
             Must include both historical and future data (i.e., length of all time series in ``data`` must be at least
-            ``prediction_length + 1``).
+            ``prediction_length + 1``, if ``cutoff`` is not provided, ``-cutoff + 1`` otherwise).
 
             The names and dtypes of columns and static features in ``data`` must match the ``train_data`` used to train
             the predictor.
 
             If provided data is a `pandas.DataFrame`, AutoGluon will attempt to convert it to a `TimeSeriesDataFrame`.
             If a `str` or a `Path` is provided, AutoGluon will attempt to load this file.
+        cutoff : int, optional
+            A _negative_ integer less than or equal to ``-1 * prediction_length`` denoting the time step in ``data``
+            where the forecast evaluation starts, i.e., time series are evaluated from the ``-cutoff``-th to the
+            ``-cutoff + prediction_length``-th time step. Defaults to ``-1 * prediction_length`, using the last
+            ``prediction_length`` time steps of each time series for evaluation.
         model : str, optional
             Name of the model that you would like to evaluate. By default, the best model during training
             (with highest validation score) will be used.
@@ -893,8 +918,10 @@ def evaluate(
             will have their signs flipped to obey this convention. For example, negative MAPE values will be reported.
             To get the ``eval_metric`` score, do ``output[predictor.eval_metric.name]``.
         """
+
         data = self._check_and_prepare_data_frame(data)
-        self._check_data_for_evaluation(data)
+        data = self._check_and_prepare_data_frame_for_evaluation(data, cutoff=cutoff)
+
         scores_dict = self._learner.evaluate(data, model=model, metrics=metrics, use_cache=use_cache)
         if display:
             logger.info("Evaluations on test data:")
@@ -1010,7 +1037,7 @@ def feature_importance(
         """
         if data is not None:
             data = self._check_and_prepare_data_frame(data)
-            self._check_data_for_evaluation(data)
+            data = self._check_and_prepare_data_frame_for_evaluation(data)
 
         fi_df = self._learner.get_feature_importance(
             data=data,
@@ -1188,6 +1215,7 @@ def unpersist(self) -> List[str]:
     def leaderboard(
         self,
         data: Optional[Union[TimeSeriesDataFrame, pd.DataFrame, Path, str]] = None,
+        cutoff: Optional[int] = None,
         extra_info: bool = False,
         extra_metrics: Optional[List[Union[str, TimeSeriesScorer]]] = None,
         display: bool = False,
@@ -1216,13 +1244,19 @@ def leaderboard(
         ----------
         data : Union[TimeSeriesDataFrame, pd.DataFrame, Path, str], optional
             dataset used for additional evaluation. Must include both historical and future data (i.e., length of all
-            time series in ``data`` must be at least ``prediction_length + 1``).
+            time series in ``data`` must be at least ``prediction_length + 1``, if ``cutoff`` is not provided,
+            ``-cutoff + 1`` otherwise).
 
             The names and dtypes of columns and static features in ``data`` must match the ``train_data`` used to train
             the predictor.
 
             If provided data is a `pandas.DataFrame`, AutoGluon will attempt to convert it to a `TimeSeriesDataFrame`.
             If a `str` or a `Path` is provided, AutoGluon will attempt to load this file.
+        cutoff : int, optional
+            A _negative_ integer less than or equal to ``-1 * prediction_length`` denoting the time step in ``data``
+            where the forecast evaluation starts, i.e., time series are evaluated from the ``-cutoff``-th to the
+            ``-cutoff + prediction_length``-th time step. Defaults to ``-1 * prediction_length`, using the last
+            ``prediction_length`` time steps of each time series for evaluation.
         extra_info : bool, default = False
             If True, the leaderboard will contain an additional column `hyperparameters` with the hyperparameters used
             by each model during training. An empty dictionary `{}` means that the model was trained with default
@@ -1258,10 +1292,12 @@ def leaderboard(
                 raise TypeError(f"TimeSeriesPredictor.leaderboard() got an unexpected keyword argument '{key}'")
         if data is None and extra_metrics is not None:
             raise ValueError("`extra_metrics` is only valid when `data` is specified.")
+        if data is None and cutoff is not None:
+            raise ValueError("`cutoff` is only valid when `data` is specified.")
 
         if data is not None:
             data = self._check_and_prepare_data_frame(data)
-            self._check_data_for_evaluation(data)
+            data = self._check_and_prepare_data_frame_for_evaluation(data, cutoff=cutoff)
 
         leaderboard = self._learner.leaderboard(
             data, extra_info=extra_info, extra_metrics=extra_metrics, use_cache=use_cache
@@ -1430,7 +1466,7 @@ def select_target(ts_df: TimeSeriesDataFrame) -> TimeSeriesDataFrame:
             return cast(TimeSeriesDataFrame, ts_df[[self.target]])
 
         test_data = self._check_and_prepare_data_frame(test_data)
-        self._check_data_for_evaluation(test_data, name="test_data")
+        test_data = self._check_and_prepare_data_frame_for_evaluation(test_data, name="test_data")
         test_data = self._learner.feature_generator.transform(test_data)
 
         trainer = self._trainer
diff --git a/timeseries/tests/unittests/test_predictor.py b/timeseries/tests/unittests/test_predictor.py
index cf3ea34cbb8..e13dac60559 100644
--- a/timeseries/tests/unittests/test_predictor.py
+++ b/timeseries/tests/unittests/test_predictor.py
@@ -1003,6 +1003,54 @@ def test_when_custom_metric_passed_to_score_then_predictor_can_evaluate(temp_mod
     assert isinstance(scores[eval_metric.name], float)
 
 
+@pytest.mark.parametrize("cutoff, prediction_length", [(-8, 9), (-9.0, 9), (9, 9), ("2020-01-01", 9)])
+def test_given_invalid_cutoff_when_evaluate_called_then_exception_is_raised(
+    temp_model_path, cutoff, prediction_length
+):
+    predictor = TimeSeriesPredictor(path=temp_model_path, prediction_length=prediction_length, freq="h")
+
+    data = get_data_frame_with_variable_lengths({"A": 30, "B": 10}, freq="h")
+    predictor.fit(data, hyperparameters={"Naive": {}})
+
+    with pytest.raises(ValueError, match="`cutoff` should be a negative integer"):
+        predictor.evaluate(data, cutoff=cutoff)
+
+
+@pytest.mark.parametrize("cutoff", [-6, -10])
+def test_metric_with_non_default_cutoff_is_different_from_metric_without_cutoff(temp_model_path, cutoff):
+    predictor = TimeSeriesPredictor(prediction_length=5, path=temp_model_path, eval_metric="MASE")
+    predictor.fit(DUMMY_TS_DATAFRAME, hyperparameters=DUMMY_HYPERPARAMETERS)
+
+    metric_cutoff = predictor.evaluate(DUMMY_TS_DATAFRAME, cutoff=cutoff)
+    metric_no_cutoff = predictor.evaluate(DUMMY_TS_DATAFRAME)
+
+    assert metric_cutoff != metric_no_cutoff
+
+    lb_cutoff = predictor.leaderboard(DUMMY_TS_DATAFRAME, cutoff=cutoff).set_index("model").sort_index()
+    lb_no_cutoff = predictor.leaderboard(DUMMY_TS_DATAFRAME).set_index("model").sort_index()
+
+    assert (lb_cutoff["score_test"] != lb_no_cutoff["score_test"]).all()
+
+
+@pytest.mark.parametrize("cutoff", [-6, -10])
+def test_metric_with_cutoff_is_same_as_slicing_and_evaluating(temp_model_path, cutoff):
+    prediction_length = 5
+    predictor = TimeSeriesPredictor(prediction_length=prediction_length, path=temp_model_path, eval_metric="MASE")
+    predictor.fit(DUMMY_TS_DATAFRAME, hyperparameters=DUMMY_HYPERPARAMETERS)
+
+    sliced_df = DUMMY_TS_DATAFRAME.slice_by_timestep(None, prediction_length + cutoff)
+
+    metric_cutoff = predictor.evaluate(DUMMY_TS_DATAFRAME, cutoff=cutoff)
+    metric_sliced = predictor.evaluate(sliced_df)
+
+    assert metric_cutoff == metric_sliced
+
+    lb_cutoff = predictor.leaderboard(DUMMY_TS_DATAFRAME, cutoff=cutoff).set_index("model").sort_index()
+    lb_sliced = predictor.leaderboard(sliced_df).set_index("model").sort_index()
+
+    assert (lb_cutoff["score_test"] == lb_sliced["score_test"]).all()
+
+
 @pytest.mark.parametrize(
     "fit_metric, metrics_passed_to_eval, expected_keys",
     [


Code review comments:
diff_hunk: @@ -844,6 +861,7 @@ def predict(
     def evaluate(
         self,
         data: Union[TimeSeriesDataFrame, pd.DataFrame, Path, str],
+        cutoff: Optional[int] = None,
path: timeseries/src/autogluon/timeseries/predictor.py
text: @user1:
Very minor, but maybe put `cutoff` after the `model` argument? I can imagine that some users call the method as `predictor.evaluate(data, "ETS")`, and we don't want to introduce a breaking change for them.

@author:
I moved it after `metrics`.