Repository: sktime/sktime

Issue:
[BUG] forecaster cutoff does not update when data requires vectorization
Forecasters do not update the `cutoff` when the data is vectorized in the hierarchical or panel case.

Example:

```python
from sktime.forecasting.arima import ARIMA
from sktime.datatypes import get_examples

y = get_examples("pd_multiindex_hier")[0]
f = ARIMA()
f.fit(y, fh=1)

f.cutoff  # prints nothing
type(f.cutoff)  # it's None
```

Expected behaviour: `f.cutoff` should be `2` and type should be `int`.

The reason for this is the `get_cutoff` utillity which returns `None` if an unexpected type is passed, in this case `VectorizedDF` that `y_inner` is in case of vectorization.

The bug is addressed in https://github.com/alan-turing-institute/sktime/pull/2870.



Discussion:

Resolving patch:
diff --git a/sktime/datatypes/_utilities.py b/sktime/datatypes/_utilities.py
--- a/sktime/datatypes/_utilities.py
+++ b/sktime/datatypes/_utilities.py
@@ -68,6 +68,7 @@ def get_index_for_series(obj, cutoff=0):
     cutoff : int, or pd.datetime, optional, default=0
         current cutoff, used to offset index if obj is np.ndarray
 
+    Returns
     -------
     index : pandas.Index, index for obj
     """
@@ -77,7 +78,28 @@ def get_index_for_series(obj, cutoff=0):
     return pd.RangeIndex(cutoff, cutoff + obj.shape[0])
 
 
-def get_cutoff(obj, cutoff=0, return_index=False):
+GET_CUTOFF_SUPPORTED_MTYPES = [
+    "pd.DataFrame",
+    "pd.Series",
+    "np.ndarray",
+    "pd-multiindex",
+    "numpy3D",
+    "nested_univ",
+    "df-list",
+    "pd_multiindex_hier",
+    "np.ndarray",
+    "numpy3D",
+]
+
+
+def get_cutoff(
+    obj,
+    cutoff=0,
+    return_index=False,
+    reverse_order=False,
+    check_input=False,
+    convert_input=True,
+):
     """Get cutoff = latest time point of time series or time series panel.
 
     Assumptions on obj are not checked, these should be validated separately.
@@ -86,10 +108,9 @@ def get_cutoff(obj, cutoff=0, return_index=False):
     Parameters
     ----------
     obj : sktime compatible time series data container
-        must be of one of the following mtypes:
-            pd.Series, pd.DataFrame, np.ndarray, of Series scitype
-            pd.multiindex, numpy3D, nested_univ, df-list, of Panel scitype
-            pd_multiindex_hier, of Hierarchical scitype
+        must be of Series, Panel, or Hierarchical scitype
+        all mtypes are supported via conversion to internally supported types
+        to avoid conversions, pass data in one of GET_CUTOFF_SUPPORTED_MTYPES
     cutoff : int, optional, default=0
         current cutoff, used to offset index if obj is np.ndarray
     return_index : bool, optional, default=False
@@ -97,12 +118,38 @@ def get_cutoff(obj, cutoff=0, return_index=False):
             or a pandas compatible index element (False)
         note: return_index=True may set freq attribute of time types to None
             return_index=False will typically preserve freq attribute
+    reverse_order : bool, optional, default=False
+        if False, returns largest time index. If True, returns smallest time index
+    check_input : bool, optional, default=False
+        whether to check input for validity, i.e., is it one of the scitypes
+    convert_input : bool, optional, default=True
+        whether to convert the input (True), or skip conversion (False)
+        if skipped, function assumes that inputs are one of GET_CUTOFF_SUPPORTED_MTYPES
 
     Returns
     -------
     cutoff_index : pandas compatible index element (if return_index=False)
         pd.Index of length 1 (if return_index=True)
+
+    Raises
+    ------
+    ValueError, TypeError, if check_input or convert_input are True
+        exceptions from check or conversion failure, in check_is_scitype, convert_to
     """
+    from sktime.datatypes import check_is_scitype, convert_to
+
+    # deal with VectorizedDF
+    if hasattr(obj, "X"):
+        obj = obj.X
+
+    if check_input:
+        valid = check_is_scitype(obj, scitype=["Series", "Panel", "Hierarchical"])
+        if not valid:
+            raise ValueError("obj must be of Series, Panel, or Hierarchical scitype")
+
+    if convert_input:
+        obj = convert_to(obj, GET_CUTOFF_SUPPORTED_MTYPES)
+
     if cutoff is None:
         cutoff = 0
 
@@ -115,26 +162,36 @@ def get_cutoff(obj, cutoff=0, return_index=False):
             cutoff_ind = obj.shape[-1] + cutoff
         if obj.ndim < 3 and obj.ndim > 0:
             cutoff_ind = obj.shape[0] + cutoff
+        if reverse_order:
+            cutoff_ind = 0
         if return_index:
             return pd.RangeIndex(cutoff_ind - 1, cutoff_ind)
         else:
             return cutoff_ind
 
+    # define "first" or "last" index depending on which is desired
+    if reverse_order:
+        ix = 0
+        agg = min
+    else:
+        ix = -1
+        agg = max
+
     if isinstance(obj, pd.Series):
-        return obj.index[[-1]] if return_index else obj.index[-1]
+        return obj.index[[ix]] if return_index else obj.index[ix]
 
     # nested_univ (Panel) or pd.DataFrame(Series)
     if isinstance(obj, pd.DataFrame) and not isinstance(obj.index, pd.MultiIndex):
         objcols = [x for x in obj.columns if obj.dtypes[x] == "object"]
         # pd.DataFrame
         if len(objcols) == 0:
-            return obj.index[[-1]] if return_index else obj.index[-1]
+            return obj.index[[ix]] if return_index else obj.index[ix]
         # nested_univ
         else:
             if return_index:
-                idxx = [x.index[[-1]] for col in objcols for x in obj[col]]
+                idxx = [x.index[[ix]] for col in objcols for x in obj[col]]
             else:
-                idxx = [x.index[-1] for col in objcols for x in obj[col]]
+                idxx = [x.index[ix] for col in objcols for x in obj[col]]
             return max(idxx)
 
     # pd-multiindex (Panel) and pd_multiindex_hier (Hierarchical)
@@ -145,15 +202,15 @@ def get_cutoff(obj, cutoff=0, return_index=False):
             cutoffs = [x[[-1]] for x in series_idx]
         else:
             cutoffs = [x[-1] for x in series_idx]
-        return max(cutoffs)
+        return agg(cutoffs)
 
     # df-list (Panel)
     if isinstance(obj, list):
         if return_index:
-            idxs = [x.index[[-1]] for x in obj]
+            idxs = [x.index[[ix]] for x in obj]
         else:
-            idxs = [x.index[-1] for x in obj]
-        return max(idxs)
+            idxs = [x.index[ix] for x in obj]
+        return agg(idxs)
 
 
 def update_data(X, X_new=None):
@@ -210,7 +267,7 @@ def update_data(X, X_new=None):
         return X_new.combine_first(X)
 
 
-GET_LATEST_WINDOW_SUPPORTED_MTYPES = [
+GET_WINDOW_SUPPORTED_MTYPES = [
     "pd.DataFrame",
     "pd-multiindex",
     "pd_multiindex_hier",
@@ -230,16 +287,17 @@ def get_window(obj, window_length=None, lag=None):
     Parameters
     ----------
     obj : sktime compatible time series data container or None
-        if not None, must be of one of the following mtypes:
-            pd.Series, pd.DataFrame, np.ndarray, of Series scitype
-            pd.multiindex, numpy3D, nested_univ, df-list, of Panel scitype
-            pd_multiindex_hier, of Hierarchical scitype
+        if not None, must be of Series, Panel, or Hierarchical scitype
+        all mtypes are supported via conversion to internally supported types
+        to avoid conversions, pass data in one of GET_WINDOW_SUPPORTED_MTYPES
     window_length : int or timedelta, optional, default=-inf
         must be int if obj is int indexed, timedelta if datetime indexed
         length of the window to slice to. Default = window of infinite size
-    lag : int or timedelta, optional, default = 0
-        must be int if obj is int indexed, timedelta if datetime indexed
+    lag : int, timedelta, or None optional, default = None (zero of correct type)
         lag of the latest time in the window, with respect to cutoff of obj
+        if None, is internally replaced by a zero of type compatible with obj index
+        must be int if obj is int indexed or not pandas based
+        must be timedelta if obj is pandas based and datetime indexed
 
     Returns
     -------
@@ -259,7 +317,7 @@ def get_window(obj, window_length=None, lag=None):
         raise ValueError("obj must be of Series, Panel, or Hierarchical scitype")
     obj_in_mtype = metadata["mtype"]
 
-    obj = convert_to(obj, GET_LATEST_WINDOW_SUPPORTED_MTYPES)
+    obj = convert_to(obj, GET_WINDOW_SUPPORTED_MTYPES)
 
     # numpy3D (Panel) or np.npdarray (Series)
     if isinstance(obj, np.ndarray):
@@ -325,11 +383,10 @@ def get_slice(obj, start=None, end=None):
 
     Parameters
     ----------
-    obj : sktime compatible time series Series type or None
-        if not None, must be of one of the following mtypes:
-            pd.Series, pd.DataFrame, np.ndarray, of Series scitype
-            pd.multiindex, numpy3D, nested_univ, df-list, of Panel scitype
-            pd_multiindex_hier, of Hierarchical scitype
+    obj : sktime compatible time series data container or None
+        if not None, must be of Series, Panel, or Hierarchical scitype
+        all mtypes are supported via conversion to internally supported types
+        to avoid conversions, pass data in one of GET_WINDOW_SUPPORTED_MTYPES
     start : int or timestamp, optional, default = None
         must be int if obj is int indexed, timestamp if datetime indexed
         Inclusive start of slice. Default = None.
@@ -355,7 +412,7 @@ def get_slice(obj, start=None, end=None):
         raise ValueError("obj must be of Series, Panel, or Hierarchical scitype")
     obj_in_mtype = metadata["mtype"]
 
-    obj = convert_to(obj, GET_LATEST_WINDOW_SUPPORTED_MTYPES)
+    obj = convert_to(obj, GET_WINDOW_SUPPORTED_MTYPES)
 
     # numpy3D (Panel) or np.npdarray (Series)
     # Assumes the index is integer so will be exclusive by default


Test patch:
diff --git a/sktime/datatypes/tests/test_utils.py b/sktime/datatypes/tests/test_utils.py
--- a/sktime/datatypes/tests/test_utils.py
+++ b/sktime/datatypes/tests/test_utils.py
@@ -23,9 +23,11 @@
 ]
 
 
+@pytest.mark.parametrize("convert_input", [True, False])
+@pytest.mark.parametrize("reverse_order", [True, False])
 @pytest.mark.parametrize("return_index", [True, False])
 @pytest.mark.parametrize("scitype,mtype", SCITYPE_MTYPE_PAIRS)
-def test_get_cutoff(scitype, mtype, return_index):
+def test_get_cutoff(scitype, mtype, return_index, reverse_order, convert_input):
     """Tests that conversions for scitype agree with from/to example fixtures.
 
     Parameters
@@ -33,6 +35,8 @@ def test_get_cutoff(scitype, mtype, return_index):
     scitype : str - scitype of input
     mtype : str - mtype of input
     return_index : bool - whether index (True) or index element is returned (False)
+    reverse_order : bool - whether first (True) or last index (False) is retrieved
+    convert_input : bool, - whether input is converted (True) or passed through (False)
 
     Raises
     ------
@@ -46,7 +50,12 @@ def test_get_cutoff(scitype, mtype, return_index):
         if fixture is None:
             continue
 
-        cutoff = get_cutoff(fixture, return_index=return_index)
+        cutoff = get_cutoff(
+            fixture,
+            return_index=return_index,
+            reverse_order=reverse_order,
+            convert_input=convert_input,
+        )
 
         if return_index:
             expected_types = pd.Index
@@ -64,6 +73,22 @@ def test_get_cutoff(scitype, mtype, return_index):
             assert len(cutoff) == 1
 
 
+@pytest.mark.parametrize("bad_inputs", ["foo", 12345, [[[]]]])
+def test_get_cutoff_wrong_input(bad_inputs):
+    """Tests that get_cutoff raises error on bad input when input checks are enabled.
+
+    Parameters
+    ----------
+    bad_inputs : inputs that should set off the input checks
+
+    Raises
+    ------
+    Exception (from pytest) if the error is not raised as expected
+    """
+    with pytest.raises(Exception, match="must be of Series, Panel, or Hierarchical"):
+        get_cutoff(bad_inputs, check_input=True)
+
+
 @pytest.mark.parametrize("window_length, lag", [(2, 0), (None, 0), (4, 1)])
 @pytest.mark.parametrize("scitype,mtype", SCITYPE_MTYPE_PAIRS)
 def test_get_window_output_type(scitype, mtype, window_length, lag):
diff --git a/sktime/forecasting/base/tests/test_base.py b/sktime/forecasting/base/tests/test_base.py
--- a/sktime/forecasting/base/tests/test_base.py
+++ b/sktime/forecasting/base/tests/test_base.py
@@ -10,6 +10,7 @@
 import pytest
 
 from sktime.datatypes import check_is_mtype, convert
+from sktime.datatypes._utilities import get_cutoff
 from sktime.forecasting.arima import ARIMA
 from sktime.utils._testing.hierarchical import _make_hierarchical
 from sktime.utils._testing.panel import _make_panel
@@ -30,7 +31,8 @@ def test_vectorization_series_to_panel(mtype):
 
     y = _make_panel(n_instances=n_instances, random_state=42, return_mtype=mtype)
 
-    y_pred = ARIMA().fit(y).predict([1, 2, 3])
+    f = ARIMA()
+    y_pred = f.fit(y).predict([1, 2, 3])
     valid, _, metadata = check_is_mtype(y_pred, mtype, return_metadata=True)
 
     msg = (
@@ -55,6 +57,12 @@ def test_vectorization_series_to_panel(mtype):
     )
     assert y_pred_equal_length, msg
 
+    msg = (
+        "estimator in vectorization test does not properly update cutoff, "
+        f"expected {y}, but found {f.cutoff}"
+    )
+    assert f.cutoff == get_cutoff(y), msg
+
 
 @pytest.mark.parametrize("mtype", HIER_MTYPES)
 def test_vectorization_series_to_hier(mtype):
@@ -69,7 +77,8 @@ def test_vectorization_series_to_hier(mtype):
     y = _make_hierarchical(hierarchy_levels=hierarchy_levels, random_state=84)
     y = convert(y, from_type="pd_multiindex_hier", to_type=mtype)
 
-    y_pred = ARIMA().fit(y).predict([1, 2, 3])
+    f = ARIMA()
+    y_pred = f.fit(y).predict([1, 2, 3])
     valid, _, metadata = check_is_mtype(y_pred, mtype, return_metadata=True)
 
     msg = (
@@ -94,6 +103,12 @@ def test_vectorization_series_to_hier(mtype):
     )
     assert y_pred_equal_length, msg
 
+    msg = (
+        "estimator in vectorization test does not properly update cutoff, "
+        f"expected {y}, but found {f.cutoff}"
+    )
+    assert f.cutoff == get_cutoff(y), msg
+
 
 PROBA_DF_METHODS = ["predict_interval", "predict_quantiles", "predict_var"]