Repository: sktime/sktime
Pull request #6451 (2024-05-19 23:38:17)

[ENH] make `get_packages_with_changed_specs` safe to mutation of return
If an `lru_cache`-d function returns a mutable object, and that object is later muted, the cached function wlil continue returning the muted - incorrect - return.

This PR fixes the one instance in the code base where this issue could arise.


Patch:
diff --git a/docs/source/api_reference/split.rst b/docs/source/api_reference/split.rst
index ee1b27fb7..742ce6c0b 100644
--- a/docs/source/api_reference/split.rst
+++ b/docs/source/api_reference/split.rst
@@ -47,6 +47,7 @@ They have tag ``"split_type"="temporal"``.
     SingleWindowSplitter
     SlidingWindowSplitter
     ExpandingWindowSplitter
+    ExpandingCutoffSplitter
     ExpandingGreedySplitter
     TemporalTrainTestSplitter
 
diff --git a/docs/source/developer_guide/deprecation.rst b/docs/source/developer_guide/deprecation.rst
index 74eabe412..4d0db2c4f 100644
--- a/docs/source/developer_guide/deprecation.rst
+++ b/docs/source/developer_guide/deprecation.rst
@@ -4,13 +4,44 @@
 Deprecation
 ===========
 
-Before we can make changes to sktime's user interface, we need to make sure that users have time to make the necessary adjustments in their code.
-For this reason, we first need to deprecate functionality and change it only in a next release.
+``sktime`` aims to be stable and reliable towards its users.
+Our high-level policy to ensure this is:
 
-.. note::
+"``sktime`` should never break user code without a clear and actionable warning
+given at least one (MINOR) release cycle in advance."
+
+Here, "break" expressly includes a change to abstract logic, such as the algorithm
+being used, not just changes that lead to exceptions or performance degradation.
+
+For instance, if a user has code
+
+.. code:: python
+
+    from sktime.forecasting.foo import BarForecaster
+
+    bar = BarForecaster(42, x=43)
+    bar.fit(y_train, fh=[1, 2, 3])
+    y_pred = bar.predict()
+
+then no release of ``sktime`` should change, without warning:
+
+* import location of ``BarForecaster``
+* argument signature of ``BarForecaster``, including name, order, and defaults of arguments
+* the abstract algorithm that ``BarForecaster`` carries out for the given arguments
+
+Changes that can be carried out without warning:
+
+* adding more arguments at the end of the argument list, with a default value that retains prior behaviour,
+  as long as the new arguments are well-documented
+* pure refactoring of internal code, as long as the public API remains the same
+* changing the implementation without changing the abstract algorithm, e.g., for performance reasons
+
+The deprecation policy outlined in this document provides details on how to carry out
+changes that need change or deprecation handling, in a user-friendly and reliable way.
+
+It is accompanied by formulaic patterns for developers, with examples,
+and a process for release managers, to make the policy easy to follow.
 
-    For upcoming changes and next releases, see our `Milestones <https://github.com/sktime/sktime/milestones?direction=asc&sort=due_date&state=open>`_.
-    For our long-term plan, see our :ref:`roadmap`.
 
 Deprecation policy
 ==================
diff --git a/pyproject.toml b/pyproject.toml
index e7a336ab5..ae8a7e8fb 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -95,7 +95,7 @@ all_extras = [
   'numba<0.60,>=0.53',
   'pmdarima!=1.8.1,<3.0.0,>=1.8; python_version < "3.12"',
   'prophet>=1.1; python_version < "3.12"',
-  "pycatch22<0.4.4",
+  "pycatch22<0.4.6",
   "pykalman<0.10,>=0.9.5",
   'pyod>=0.8; python_version < "3.11"',
   "pyts<0.14.0; python_version < '3.12'",
@@ -136,7 +136,7 @@ all_extras_pandas2 = [
   'numba<0.60,>=0.53',
   'pmdarima!=1.8.1,<3.0.0,>=1.8; python_version < "3.12"',
   'prophet>=1.1; python_version < "3.12"',
-  "pycatch22<0.4.4",
+  "pycatch22<0.4.6",
   "pykalman<0.10,>=0.9.5",
   'pyod>=0.8; python_version < "3.11"',
   "scikit_posthocs>=0.6.5",
@@ -207,7 +207,7 @@ regression = [
 transformations = [
   'esig<0.10,>=0.9.7; python_version < "3.11"',
   "filterpy<1.5,>=1.4.5",
-  "holidays>=0.29,<0.49",
+  "holidays>=0.29,<0.50",
   "mne>=1.5,<1.8",
   'numba<0.60,>=0.53',
   "pycatch22>=0.4,<0.4.6",
diff --git a/sktime/networks/base.py b/sktime/networks/base.py
index 7d04a18fb..4cf14fff7 100644
--- a/sktime/networks/base.py
+++ b/sktime/networks/base.py
@@ -4,15 +4,7 @@ __author__ = ["Withington", "TonyBagnall"]
 
 from abc import ABC, abstractmethod
 
-import numpy as np
-import pandas as pd
-
 from sktime.base import BaseObject
-from sktime.forecasting.base import BaseForecaster
-from sktime.utils.validation._dependencies import _check_soft_dependencies
-
-if _check_soft_dependencies("torch", severity="none"):
-    import torch
 
 
 class BaseDeepNetwork(BaseObject, ABC):
@@ -35,244 +27,3 @@ class BaseDeepNetwork(BaseObject, ABC):
         output_layer : a keras layer
         """
         ...
-
-
-class BaseDeepNetworkPyTorch(BaseForecaster, ABC):
-    """Abstract base class for deep learning networks using torch.nn."""
-
-    _tags = {
-        "python_dependencies": "torch",
-        "y_inner_mtype": "pd.DataFrame",
-        "capability:insample": False,
-        "capability:pred_int:insample": False,
-        "scitype:y": "both",
-        "ignores-exogeneous-X": True,
-    }
-
-    def __init__(
-        self,
-        num_epochs=16,
-        batch_size=8,
-        in_channels=1,
-        individual=False,
-        criterion_kwargs=None,
-        optimizer=None,
-        optimizer_kwargs=None,
-        lr=0.001,
-    ):
-        self.num_epochs = num_epochs
-        self.batch_size = batch_size
-        self.in_channels = in_channels
-        self.individual = individual
-        self.criterion_kwargs = criterion_kwargs
-        self.optimizer = optimizer
-        self.optimizer_kwargs = optimizer_kwargs
-        self.lr = lr
-
-        super().__init__()
-
-    def _fit(self, y, fh, X=None):
-        """Fit the network.
-
-        Changes to state:
-            writes to self._network.state_dict
-
-        Parameters
-        ----------
-        X : iterable-style or map-style dataset
-            see (https://pytorch.org/docs/stable/data.html) for more information
-        """
-        from sktime.forecasting.base import ForecastingHorizon
-
-        # save fh and y for prediction later
-        if fh.is_relative:
-            self._fh = fh
-        else:
-            fh = fh.to_relative(self.cutoff)
-            self._fh = fh
-
-        self._y = y
-
-        if type(fh) is ForecastingHorizon:
-            self.network = self._build_network(fh._values[-1])
-        else:
-            self.network = self._build_network(fh)
-
-        if self.criterion:
-            if self.criterion in self.criterions.keys():
-                if self.criterion_kwargs:
-                    self._criterion = self.criterions[self.criterion](
-                        **self.criterion_kwargs
-                    )
-                else:
-                    self._criterion = self.criterions[self.criterion]()
-            else:
-                raise TypeError(
-                    f"Please pass one of {self.criterions.keys()} for `criterion`."
-                )
-        else:
-            # default criterion
-            self._criterion = torch.nn.MSELoss()
-
-        if self.optimizer:
-            if self.optimizer in self.optimizers.keys():
-                if self.optimizer_kwargs:
-                    self._optimizer = self.optimizers[self.optimizer](
-                        self.network.parameters(), lr=self.lr, **self.optimizer_kwargs
-                    )
-                else:
-                    self._optimizer = self.optimizers[self.optimizer](
-                        self.network.parameters(), lr=self.lr
-                    )
-            else:
-                raise TypeError(
-                    f"Please pass one of {self.optimizers.keys()} for `optimizer`."
-                )
-        else:
-            # default optimizer
-            self._optimizer = torch.optim.Adam(self.network.parameters(), lr=self.lr)
-
-        dataloader = self.build_pytorch_train_dataloader(y)
-        self.network.train()
-
-        for _ in range(self.num_epochs):
-            for x, y in dataloader:
-                y_pred = self.network(x)
-                loss = self._criterion(y_pred, y)
-                self._optimizer.zero_grad()
-                loss.backward()
-                self._optimizer.step()
-
-    def _predict(self, X=None, fh=None):
-        """Predict with fitted model."""
-        from torch import cat
-
-        if fh is None:
-            fh = self._fh
-
-        if max(fh._values) > self.network.pred_len or min(fh._values) < 0:
-            raise ValueError(
-                f"fh of {fh} passed to {self.__class__.__name__} is not "
-                "within `pred_len`. Please use a fh that aligns with the `pred_len` of "
-                "the forecaster."
-            )
-
-        if X is None:
-            dataloader = self.build_pytorch_pred_dataloader(self._y, fh)
-        else:
-            dataloader = self.build_pytorch_pred_dataloader(X, fh)
-
-        y_pred = []
-        for x, _ in dataloader:
-            y_pred.append(self.network(x).detach())
-        y_pred = cat(y_pred, dim=0).view(-1, y_pred[0].shape[-1]).numpy()
-        y_pred = y_pred[fh._values.values - 1]
-        y_pred = pd.DataFrame(
-            y_pred, columns=self._y.columns, index=fh.to_absolute_index(self.cutoff)
-        )
-
-        return y_pred
-
-    def build_pytorch_train_dataloader(self, y):
-        """Build PyTorch DataLoader for training."""
-        from torch.utils.data import DataLoader
-
-        if self.custom_dataset_train:
-            if hasattr(self.custom_dataset_train, "build_dataset") and callable(
-                self.custom_dataset_train.build_dataset
-            ):
-                self.custom_dataset_train.build_dataset(y)
-                dataset = self.custom_dataset_train
-            else:
-                raise NotImplementedError(
-                    "Custom Dataset `build_dataset` method is not available. Please "
-                    f"refer to the {self.__class__.__name__}.build_dataset "
-                    "documentation."
-                )
-        else:
-            dataset = PyTorchTrainDataset(
-                y=y,
-                seq_len=self.network.seq_len,
-                fh=self._fh._values[-1],
-            )
-
-        return DataLoader(
-            dataset,
-            self.batch_size,
-        )
-
-    def build_pytorch_pred_dataloader(self, y, fh):
-        """Build PyTorch DataLoader for prediction."""
-        from torch.utils.data import DataLoader
-
-        if self.custom_dataset_pred:
-            if hasattr(self.custom_dataset_pred, "build_dataset") and callable(
-                self.custom_dataset_pred.build_dataset
-            ):
-                self.custom_dataset_train.build_dataset(y)
-                dataset = self.custom_dataset_train
-            else:
-                raise NotImplementedError(
-                    "Custom Dataset `build_dataset` method is not available. Please"
-                    f"refer to the {self.__class__.__name__}.build_dataset"
-                    "documentation."
-                )
-        else:
-            dataset = PyTorchPredDataset(
-                y=y[-self.network.seq_len :],
-                seq_len=self.network.seq_len,
-            )
-
-        return DataLoader(
-            dataset,
-            self.batch_size,
-        )
-
-    def get_y_true(self, y):
-        """Get y_true values for validation."""
-        dataloader = self.build_pytorch_pred_dataloader(y)
-        y_true = [y.flatten().numpy() for _, y in dataloader]
-        return np.concatenate(y_true, axis=0)
-
-
-class PyTorchTrainDataset:
-    """Dataset for use in sktime deep learning forecasters."""
-
-    def __init__(self, y, seq_len, fh):
-        self.y = y.values
-        self.seq_len = seq_len
-        self.fh = fh
-
-    def __len__(self):
-        """Return length of dataset."""
-        return len(self.y) - self.seq_len - self.fh + 1
-
-    def __getitem__(self, i):
-        """Return data point."""
-        from torch import from_numpy, tensor
-
-        return (
-            tensor(self.y[i : i + self.seq_len]).float(),
-            from_numpy(self.y[i + self.seq_len : i + self.seq_len + self.fh]).float(),
-        )
-
-
-class PyTorchPredDataset:
-    """Dataset for use in sktime deep learning forecasters."""
-
-    def __init__(self, y, seq_len):
-        self.y = y.values
-        self.seq_len = seq_len
-
-    def __len__(self):
-        """Return length of dataset."""
-        return 1
-
-    def __getitem__(self, i):
-        """Return data point."""
-        from torch import from_numpy, tensor
-
-        return (
-            tensor(self.y[i : i + self.seq_len]).float(),
-            from_numpy(self.y[i + self.seq_len : i + self.seq_len]).float(),
-        )
diff --git a/sktime/performance_metrics/forecasting/probabilistic/__init__.py b/sktime/performance_metrics/forecasting/probabilistic/__init__.py
index 5d17073e0..fbab9f8df 100644
--- a/sktime/performance_metrics/forecasting/probabilistic/__init__.py
+++ b/sktime/performance_metrics/forecasting/probabilistic/__init__.py
@@ -17,6 +17,7 @@ __all__ = [
     "EmpiricalCoverage",
     "ConstraintViolation",
     "CRPS",
+    "IntervalWidth",
     "LogLoss",
     "SquaredDistrLoss",
 ]
@@ -25,6 +26,7 @@ from sktime.performance_metrics.forecasting.probabilistic._classes import (
     CRPS,
     ConstraintViolation,
     EmpiricalCoverage,
+    IntervalWidth,
     LogLoss,
     PinballLoss,
     SquaredDistrLoss,
diff --git a/sktime/performance_metrics/forecasting/probabilistic/_classes.py b/sktime/performance_metrics/forecasting/probabilistic/_classes.py
index 42d727677..cfee5b76f 100644
--- a/sktime/performance_metrics/forecasting/probabilistic/_classes.py
+++ b/sktime/performance_metrics/forecasting/probabilistic/_classes.py
@@ -599,6 +599,70 @@ class EmpiricalCoverage(_BaseProbaForecastingErrorMetric):
         return [params1]
 
 
+class IntervalWidth(_BaseProbaForecastingErrorMetric):
+    """Interval width for interval predictions, sometimes also known as sharpness.
+
+    Parameters
+    ----------
+    multioutput : string "uniform_average" or "raw_values" determines how\
+        multioutput results will be treated.
+
+    score_average : bool, optional, default = True
+        specifies whether scores for each quantile should be averaged.
+    """
+
+    _tags = {
+        "scitype:y_pred": "pred_interval",
+        "lower_is_better": True,
+    }
+
+    def __init__(self, multioutput="uniform_average", score_average=True):
+        self.score_average = score_average
+        self.multioutput = multioutput
+        super().__init__(score_average=score_average, multioutput=multioutput)
+
+    def _evaluate_by_index(self, y_true, y_pred, multioutput, **kwargs):
+        """Logic for finding the metric evaluated at each index.
+
+        y_true : pd.Series, pd.DataFrame or np.array of shape (fh,) or \
+            (fh, n_outputs) where fh is the forecasting horizon
+            Ground truth (correct) target values.
+
+        y_pred : pd.Series, pd.DataFrame or np.array of shape (fh,) or  \
+            (fh, n_outputs)  where fh is the forecasting horizon
+            Forecasted values.
+
+        multioutput : string "uniform_average" or "raw_values" determines how \
+            multioutput results will be treated.
+        """
+        lower = y_pred.iloc[:, y_pred.columns.get_level_values(2) == "lower"].to_numpy()
+        upper = y_pred.iloc[:, y_pred.columns.get_level_values(2) == "upper"].to_numpy()
+
+        if not isinstance(y_true, np.ndarray):
+            y_true_np = y_true.to_numpy()
+        else:
+            y_true_np = y_true
+        if y_true_np.ndim == 1:
+            y_true_np = y_true.reshape(-1, 1)
+
+        scores = np.unique(np.round(y_pred.columns.get_level_values(1), 7))
+        vars = np.unique(y_pred.columns.get_level_values(0))
+
+        metric_array = upper - lower
+
+        out_df = pd.DataFrame(
+            metric_array, columns=pd.MultiIndex.from_product([vars, scores])
+        )
+
+        return out_df
+
+    @classmethod
+    def get_test_params(self):
+        """Retrieve test parameters."""
+        params1 = {}
+        return [params1]
+
+
 class ConstraintViolation(_BaseProbaForecastingErrorMetric):
     """Percentage of interval constraint violations for interval predictions.
 
diff --git a/sktime/split/__init__.py b/sktime/split/__init__.py
index 012fc952a..6c1c2e630 100644
--- a/sktime/split/__init__.py
+++ b/sktime/split/__init__.py
@@ -3,6 +3,7 @@
 __all__ = [
     "CutoffSplitter",
     "CutoffFhSplitter",
+    "ExpandingCutoffSplitter",
     "ExpandingGreedySplitter",
     "ExpandingWindowSplitter",
     "InstanceSplitter",
@@ -15,6 +16,7 @@ __all__ = [
 ]
 
 from sktime.split.cutoff import CutoffFhSplitter, CutoffSplitter
+from sktime.split.expandingcutoff import ExpandingCutoffSplitter
 from sktime.split.expandinggreedy import ExpandingGreedySplitter
 from sktime.split.expandingwindow import ExpandingWindowSplitter
 from sktime.split.instance import InstanceSplitter
diff --git a/sktime/split/expandingcutoff.py b/sktime/split/expandingcutoff.py
new file mode 100644
index 000000000..c4ca15d9a
--- /dev/null
+++ b/sktime/split/expandingcutoff.py
@@ -0,0 +1,213 @@
+#!/usr/bin/env python3 -u
+# copyright: sktime developers, BSD-3-Clause License (see LICENSE file)
+"""Splitter that successively expands the training window."""
+
+__author__ = ["ninedigits"]
+
+__all__ = [
+    "ExpandingCutoffSplitter",
+]
+
+from typing import Optional
+
+import numpy as np
+import pandas as pd
+
+from sktime.split.base import BaseSplitter
+from sktime.split.base._common import ACCEPTED_Y_TYPES, _check_fh
+from sktime.utils.validation.forecasting import check_step_length
+
+
+def _is_datetimelike(x):
+    return isinstance(x, (pd.Timestamp, np.datetime64, pd.Period))
+
+
+def _is_int(arg):
+    return isinstance(arg, (int, np.int_))
+
+
+def _validate_cutoff(cutoff):
+    if not (_is_datetimelike(cutoff) or _is_int(cutoff)):
+        raise TypeError(
+            "Cutoff value must be datetime-like or integer but instead "
+            f"found type(cutoff) = {type(cutoff)}"
+        )
+    return cutoff
+
+
+class ExpandingCutoffSplitter(BaseSplitter):
+    """
+    Expanding cutoff splitter for time series data.
+
+    This splitter combines elements of `ExpandingWindowSplitter` and `CutoffSplitter`
+    to create training and testing sets. Unlike `ExpandingWindowSplitter` which begins
+    with a fixed initial window, this splitter uses a specific cutoff point as the
+    starting window for the training set. The training set then expands incrementally
+    in each split until it reaches the end of the series.
+
+    The test set is defined by a forecast horizon relative to the last point in the
+    training set, containing as many subsequent indices as specified by the `fh`
+    parameter.
+
+    The valid types of y-index and cutoff pairings are datelike-datelike, datelike-int,
+    and int-int. When a datelike index is combined with an int cutoff, the cutoff
+    functions as an iloc indexer. When an int index is paired with a positive int
+    cutoff, the cutoff serves as a loc indexer. If the int cutoff is negative, it
+    functions as an iloc indexer.
+                          c
+                          |
+    |---------------------|----fh----|------|
+    | * * * * * * * * * * x x x x x x - - - |
+    | * * * * * * * * * * * x x x x x x - - |
+    | * * * * * * * * * * * * x x x x x x - |
+    | * * * * * * * * * * * * * x x x x x x |
+
+    ``c`` = cutoff date or index.
+
+    ``*`` = training fold.
+
+    ``x`` = test fold.
+
+    Parameters
+    ----------
+    cutoff (int or pd.Timestamp):
+        The initial cutoff point in the series, which marks the beginning of the
+        first test set.
+    fh (int, list, or np.array):
+        Forecasting horizon, determining the size and  indices of the test sets.
+        It can be an integer, a list, or an array.
+    step_length (int):
+        The step length to expand the training set size in each split.
+
+    Examples
+    --------
+    >>> import pandas as pd
+    >>> from sktime.split import ExpandingCutoffSplitter
+    >>> date_range = pd.date_range(start='2020-Q1', end='2021-Q3', freq='QS')
+    >>> y = pd.DataFrame(index=pd.PeriodIndex(date_range, freq='Q'))
+    >>> cutoff = pd.Period('2021-Q1')
+    >>> cv = ExpandingCutoffSplitter(cutoff=cutoff, fh=[1, 2], step_length=1)
+    >>> list(cv.split(y)) # doctest: +SKIP
+    [(array([0, 1, 2, 3]), array([4, 5])), (array([0, 1, 2, 3, 4]), array([5, 6]))]
+    """
+
+    def __init__(self, cutoff, fh, step_length):
+        super().__init__()
+        self.cutoff = _validate_cutoff(cutoff)
+        self.fh = fh
+        self._fh = _check_fh(fh)
+        self.step_length = step_length
+        return
+
+    def _split(self, y, fh=None):
+        """
+        Generate indices to split data into training and testing sets.
+
+        Parameters
+        ----------
+        y : array-like, shape = [n_samples]
+            Time series data.
+        fh : int, default=None
+            Forecast horizon, if None, uses self.fh
+
+        Yields
+        ------
+        train : ndarray
+            The training set indices for that split.
+        test : ndarray
+            The testing set indices for that split.
+        """
+        if fh is None:
+            fh = self._fh
+        offset = fh.to_numpy().max() + 1
+        for cutoff in self.get_cutoffs(y):
+            train_window = np.arange(0, cutoff + 1, step=1)
+            test_window = np.arange(cutoff + 1, cutoff + offset, step=1)
+            yield train_window, test_window
+
+    def _validate_y(self, y):
+        y = self._coerce_to_index(y)
+        cutoff = self.cutoff
+        y0 = y[0]
+        cond1 = _is_datetimelike(y0) and (_is_datetimelike(cutoff) or _is_int(cutoff))
+        cond2 = _is_int(y0) and _is_int(cutoff)
+        if not (cond1 or cond2):
+            raise TypeError(
+                "Valid combinations for y and cutoff types are "
+                "datelike-datelike, datelike-int, or int-int, but instead "
+                f"found {type(y0)}-{type(cutoff)}"
+            )
+        return y
+
+    def _get_first_cutoff_index(self, y_index):
+        y0 = y_index[0]
+        if _is_int(self.cutoff) and self.cutoff < 0:
+            index = len(y_index) + self.cutoff - 1
+            if index < 0:
+                raise IndexError(
+                    f"Cutoff of value {self.cutoff} is out of bounds "
+                    f"for y of length {len(y_index)}"
+                )
+        elif _is_datetimelike(y0) and _is_int(self.cutoff) and (self.cutoff > 0):
+            index = self.cutoff - 1
+        else:
+            index = np.argmax(y_index == self.cutoff) - 1
+            if index == -1:
+                raise TypeError(
+                    "Could not find matching index, make sure that "
+                    f"type(y) {type(y0)} is compatible with type(cutoff) "
+                    f"{type(self.cutoff)}"
+                )
+        return index
+
+    def get_cutoffs(self, y: Optional[ACCEPTED_Y_TYPES] = None) -> np.ndarray:
+        """Return the cutoff points in .iloc[] context.
+
+        Parameters
+        ----------
+        y : pd.Series or pd.Index, optional (default=None)
+            Time series to split
+
+        Returns
+        -------
+        cutoffs : 1D np.ndarray of int
+            iloc location indices, in reference to y, of cutoff indices
+        """
+        if y is None:
+            raise ValueError(
+                f"{self.__class__.__name__} requires `y` to compute the cutoffs."
+            )
+        y = self._validate_y(y)
+        fh = self._fh
+        step_length = check_step_length(self.step_length)
+        cutoff_index = self._get_first_cutoff_index(y)
+        cutoffs = np.array([cutoff_index])
+        offset = fh.to_numpy().max()
+        while cutoff_index + offset < len(y) - 1:
+            cutoff_index += step_length
+            cutoffs = np.append(cutoffs, cutoff_index)
+        return cutoffs
+
+    @classmethod
+    def get_test_params(cls, parameter_set="default"):
+        """Return testing parameter settings for the splitter.
+
+        Parameters
+        ----------
+        parameter_set : str, default="default"
+            Name of the set of test parameters to return, for use in tests. If no
+            special parameters are defined for a value, will return ``"default"`` set.
+
+        Returns
+        -------
+        params : dict or list of dict, default = {}
+            Parameters to create testing instances of the class
+            Each dict are parameters to construct an "interesting" test instance, i.e.,
+            ``MyClass(**params)`` or ``MyClass(**params[i])`` creates a valid test
+            instance.
+            ``create_test_instance`` uses the first (or only) dictionary in ``params``
+        """
+        return [
+            {"fh": [2, 4], "cutoff": 3, "step_length": 1},
+            {"fh": [1, 2, 3], "cutoff": 3, "step_length": 2},
+        ]
diff --git a/sktime/utils/git_diff.py b/sktime/utils/git_diff.py
index f5cf6ef72..0d53af9ca 100644
--- a/sktime/utils/git_diff.py
+++ b/sktime/utils/git_diff.py
@@ -126,7 +126,6 @@ def get_changed_lines(file_path, only_indented=True):
         return []
 
 
-@lru_cache
 def get_packages_with_changed_specs():
     """Get packages with changed or added specs.
 
@@ -134,6 +133,20 @@ def get_packages_with_changed_specs():
     -------
     list of str : names of packages with changed or added specs
     """
+    return list(_get_packages_with_changed_specs())
+
+
+@lru_cache
+def _get_packages_with_changed_specs():
+    """Get packages with changed or added specs.
+
+    Private version of get_packages_with_changed_specs,
+    to avoid side effects on the list return.
+
+    Returns
+    -------
+    tuple of str : names of packages with changed or added specs
+    """
     from packaging.requirements import Requirement
 
     changed_lines = get_changed_lines("pyproject.toml")
@@ -161,6 +174,6 @@ def get_packages_with_changed_specs():
         packages.append(pkg)
 
     # make unique
-    packages = list(set(packages))
+    packages = tuple(set(packages))
 
     return packages
diff --git a/sktime/utils/plotting.py b/sktime/utils/plotting.py
index 534a26fa5..fd2667c4d 100644
--- a/sktime/utils/plotting.py
+++ b/sktime/utils/plotting.py
@@ -505,22 +505,46 @@ def plot_windows(cv, y, title="", ax=None):
 
 
 def plot_calibration(y_true, y_pred, ax=None):
-    """Plot the calibration of a probabilistic forecast.
+    r"""Plot the calibration curve for a sample of quantile predictions.
 
-    Calculates internally the calibration of the quantile forecast and
-    visualise it.
+    Visualizes calibration of the quantile predictions.
 
-    x-axis: interval from 0 to 1
-    y-axis: interval from 0 to 1
-    plot elements: the calibration fo the forecast (blue) and the ideal
-        calibration (orange)
+    Computes the following calibration plot:
+
+    Let :math:`p_1, \dots, p_k` be the quantile points at which
+    predictions in ``y_pred`` were queried,
+    e.g., via ``alpha`` in ``predict_quantiles``.
+
+    Let :math:`y_1, \dots, y_N` be the actual values in ``y_true``,
+    and let :math:`\widehat{y}_{i,j}`, for `i = 1 \dots N, j = 1 \dota k`
+    be quantile predictions at quantile point :math:`p_j`,
+    of the conditional distribution of :math:`y_i`, as contained in ``y_pred``.
+
+    We compute the calibration indicators :math:`c_{i, j},`
+    as :math:`c_{i, j} = 1, \{ if } y_i \le \widehat{y}_{i,j} \text{ and } 0, \text{otherwise},`  # noqa: E501
+    and calibration fractions as
+
+    .. math:: \widehat{p}_j = \frac{1}{N} \sum_{i = 1}^N c_{i, j}.
+
+    If the quantile predictions are well-calibrated, we expect :math:`\widehat{p}_j`
+    to be close to :math:`p_j`.
+
+    x-axis: interval from 0 to 1, quantile points
+    y-axis: interval from 0 to 1, calibration fractions
+    plot elements: calibration curve of the quantile predictions (blue) and the ideal
+        calibration curve (orange), the curve with equation y = x.
+        Calibration curve are points :math:`(p_i, \widehat{p}_i), i = 1 \dots, k`;
+        Ideal curve is the curve with equation y = x,
+        containing points :math:`(p_i, p_i)`.
 
     Parameters
     ----------
     y_true : pd.Series, single columned pd.DataFrame, or single columned np.array.
-        The actual values of the forecast
+        The actual values
     y_pred : pd.DataFrame
-        The quantile forecast.
+        The quantile predictions,
+        formatted as returned by ``BaseDistribution.quantile``,
+        or ``predict_quantiles``
     ax : matplotlib.axes.Axes, optional (default=None)
         Axes on which to plot. If None, axes will be created and returned.
 


Test patch:
diff --git a/sktime/performance_metrics/forecasting/probabilistic/tests/test_probabilistic_metrics.py b/sktime/performance_metrics/forecasting/probabilistic/tests/test_probabilistic_metrics.py
index 2c683a688..b5e450d51 100644
--- a/sktime/performance_metrics/forecasting/probabilistic/tests/test_probabilistic_metrics.py
+++ b/sktime/performance_metrics/forecasting/probabilistic/tests/test_probabilistic_metrics.py
@@ -8,6 +8,7 @@ import pytest
 from sktime.performance_metrics.forecasting.probabilistic import (
     ConstraintViolation,
     EmpiricalCoverage,
+    IntervalWidth,
     PinballLoss,
 )
 from sktime.split import temporal_train_test_split
@@ -20,8 +21,9 @@ quantile_metrics = [
 ]
 
 interval_metrics = [
-    EmpiricalCoverage,
     ConstraintViolation,
+    EmpiricalCoverage,
+    IntervalWidth,
 ]
 
 all_metrics = interval_metrics + quantile_metrics
diff --git a/sktime/split/tests/test_expandingcutoff.py b/sktime/split/tests/test_expandingcutoff.py
new file mode 100644
index 000000000..7c7133120
--- /dev/null
+++ b/sktime/split/tests/test_expandingcutoff.py
@@ -0,0 +1,213 @@
+# copyright: sktime developers, BSD-3-Clause License (see LICENSE file)
+"""Tests for expanding cutoff splitter."""
+
+import numpy as np
+import pandas as pd
+import pytest
+
+from sktime.forecasting.base import ForecastingHorizon
+from sktime.split import ExpandingCutoffSplitter
+from sktime.split.tests.test_split import _check_cv
+from sktime.utils._testing.hierarchical import _make_hierarchical
+from sktime.utils._testing.series import _make_series
+
+
+def test_expandingcutoff_datelike_index_001():
+    """Test datetime index with _check_cv"""
+    y = _make_series(n_timepoints=10, random_state=42)
+    cutoff = y.index[3]
+    fh = ForecastingHorizon([1, 2, 3], freq=y.index.freq)
+    cv = ExpandingCutoffSplitter(cutoff=cutoff, fh=fh, step_length=1)
+    train_windows, test_windows, cutoffs, n_splits = _check_cv(cv, y)
+    np.testing.assert_array_equal(cutoffs, cv.get_cutoffs(y))
+
+
+def test_expandingcutoff_int_index_002():
+    """Test int index with _check_cv"""
+    y = _make_series(n_timepoints=10, index_type="int", random_state=42)
+    cutoff = y.index[3]
+    fh = ForecastingHorizon([1, 2, 3])
+    cv = ExpandingCutoffSplitter(cutoff=cutoff, fh=fh, step_length=1)
+    train_windows, test_windows, cutoffs, n_splits = _check_cv(cv, y)
+    np.testing.assert_array_equal(cutoffs, cv.get_cutoffs(y))
+
+
+def test_expandingcutoff_ytype_cutofftype_combos_003a():
+    """Test invalid param combo"""
+    # Datetime cutoff
+    i = 3
+    y1 = _make_series(n_timepoints=10, index_type="datetime", random_state=42)
+    fh = ForecastingHorizon([1, 2, 3])
+    cutoffs = [-7, y1.index[i], 5]
+    step_lengths = [1, 2, 3]
+    for cutoff in cutoffs:
+        for step_length in step_lengths:
+            cv1 = ExpandingCutoffSplitter(cutoff=cutoff, fh=fh, step_length=step_length)
+            _check_cv(cv1, y1)
+
+
+def test_expandingcutoff_ytype_cutofftype_combos_003b():
+    """Test invalid param combo"""
+    # Datetime cutoff
+    y1 = _make_series(n_timepoints=10, index_type="int", random_state=42)
+    fh = ForecastingHorizon([1, 2, 3])
+    cutoffs = [-7, y1.index[3]]
+    step_lengths = [1, 2, 3]
+    for cutoff in cutoffs:
+        for step_length in step_lengths:
+            cv1 = ExpandingCutoffSplitter(cutoff=cutoff, fh=fh, step_length=step_length)
+            _check_cv(cv1, y1)
+
+
+def test_expandingcutoff_ytype_cutofftype_combos_003c():
+    """Test invalid param combo"""
+    # Datetime cutoff
+    y1 = _make_series(n_timepoints=10, index_type="int", random_state=42)
+    fh = ForecastingHorizon([1, 2, 3])
+    # Y-index is int but cutoff is datetime
+    with pytest.raises(TypeError):
+        cv1 = ExpandingCutoffSplitter(
+            cutoff=pd.Timestamp("2000-01-01"), fh=fh, step_length=1
+        )
+        _check_cv(cv1, y1)
+
+
+def test_expandingcutoff_splitloc_004():
+    """Test split loc"""
+    y = _make_series(n_timepoints=10, random_state=42)
+    cutoff = y.index[3]
+    fh = ForecastingHorizon([1, 2, 3], freq=y.index.freq)
+    cv = ExpandingCutoffSplitter(cutoff=cutoff, fh=fh, step_length=1)
+    train_test_values = list(cv.split_loc(y))
+    last_train, last_test = train_test_values[-1]
+    expected_last_train = y.index[0:7]
+    expected_last_test = y.index[7:]
+    test_cases = [(expected_last_train, last_train), (expected_last_test, last_test)]
+    for expected, actual in test_cases:
+        np.testing.assert_array_equal(expected.values, actual.values)
+
+
+def test_expandingcutoff_hiearchical_splitloc_005():
+    """Test hiearchical splitloc with datetime"""
+    y = _make_hierarchical(
+        min_timepoints=6,
+        max_timepoints=10,
+        same_cutoff=True,
+        hierarchy_levels=(1, 2),
+        index_type="datetime",
+        random_state=42,
+    )
+    y_index = y.index.get_level_values(-1)
+    cutoff_index = -4
+    cutoff = y_index[cutoff_index]
+    fh = ForecastingHorizon([1, 2, 3])
+    cv = ExpandingCutoffSplitter(cutoff=cutoff, fh=fh, step_length=1)
+
+    train_test = list(cv.split_loc(y))
+
+    y_train_expected = (
+        y.groupby(level=-2)
+        .apply(lambda row: row[0:cutoff_index])
+        .index.get_level_values(-1)
+    )
+    y_test_expected = (
+        y.groupby(level=-2)
+        .apply(lambda row: row[cutoff_index : cutoff_index + fh[-1]])
+        .index.get_level_values(-1)
+    )
+    pd.testing.assert_index_equal(
+        y_train_expected, train_test[0][0].get_level_values(-1)
+    )
+    pd.testing.assert_index_equal(
+        y_test_expected, train_test[0][1].get_level_values(-1)
+    )
+
+
+def test_expandingcutoff_hiearchical_forecastbylevel_006():
+    """Test hiearchical with forecast by level"""
+    from sktime.forecasting.compose import ForecastByLevel
+    from sktime.forecasting.model_selection import ForecastingGridSearchCV
+    from sktime.forecasting.naive import NaiveForecaster
+    from sktime.utils._testing.hierarchical import _make_hierarchical
+
+    y = _make_hierarchical(
+        min_timepoints=6,
+        max_timepoints=10,
+        same_cutoff=True,
+        hierarchy_levels=(1, 2),
+        index_type="datetime",
+        random_state=42,
+    )
+    y_index = y.index.get_level_values(-1)
+    last_date = y_index.max()
+    cutoff_index = -4
+    cutoff = y_index[cutoff_index]
+    fh = ForecastingHorizon([1, 2, 3])
+    cv = ExpandingCutoffSplitter(cutoff=cutoff, fh=fh, step_length=1)
+    forecaster = NaiveForecaster()
+    param_grid = {"strategy": ["last", "mean", "drift"]}
+    gscv = ForecastByLevel(
+        ForecastingGridSearchCV(forecaster=forecaster, param_grid=param_grid, cv=cv)
+    )
+    gscv.fit(y, fh=fh)
+    y_pred = gscv.predict()
+    expected_last_forecast_date = last_date + pd.Timedelta(days=fh[-1])
+    actual_last_forecast_date = y_pred.index.max()
+    assert expected_last_forecast_date, actual_last_forecast_date
+
+
+def test_expandingcutoff_fh_list_007():
+    """Test fh as list with _check_cv"""
+    y = _make_series(n_timepoints=10, random_state=42)
+    cutoff = y.index[3]
+    fh = [1, 2, 3]
+    cv = ExpandingCutoffSplitter(cutoff=cutoff, fh=fh, step_length=1)
+    train_windows, test_windows, cutoffs, n_splits = _check_cv(cv, y)
+    np.testing.assert_array_equal(cutoffs, cv.get_cutoffs(y))
+
+
+def test_expanding_cutoff_period_008():
+    date_range = pd.date_range(
+        start=pd.Timestamp("2020-Q1"), end=pd.Timestamp("2021-Q4"), freq="QS"
+    )
+    y = pd.DataFrame({"date": date_range})
+    y = y.reset_index()
+    y = y.set_index("date")
+    y.index = pd.PeriodIndex(y.index, freq="Q")
+
+    cutoff = pd.Period("2021-Q1")
+    fh = ForecastingHorizon([1, 2, 3])
+    cv = ExpandingCutoffSplitter(cutoff=cutoff, fh=fh, step_length=2)
+    _check_cv(cv, y)
+
+    # pandas periods and timestamps are both datelike but don't work directly
+    cutoff = pd.Timestamp("2021-Q1")
+    with pytest.raises(TypeError):
+        cv = ExpandingCutoffSplitter(cutoff=cutoff, fh=fh, step_length=1)
+        _check_cv(cv, y)
+
+
+def _make_splits_listlike(splits):
+    splits_new = []
+    for train, test in splits:
+        splits_new.append([train.tolist(), test.tolist()])
+    return splits_new
+
+
+def test_expanding_cutoff_docstring_examples():
+    date_range = pd.date_range(start="2020-Q1", end="2021-Q3", freq="QS")
+    y = pd.DataFrame(index=pd.PeriodIndex(date_range, freq="Q"))
+    cv1 = ExpandingCutoffSplitter(cutoff=pd.Period("2021-Q1"), fh=[1, 2], step_length=1)
+    splits1 = list(cv1.split(y))
+
+    cv2 = ExpandingCutoffSplitter(cutoff=4, fh=[1, 2], step_length=1)
+    splits2 = list(cv2.split(y))
+
+    cv3 = ExpandingCutoffSplitter(cutoff=-3, fh=[1, 2], step_length=1)
+    splits3 = list(cv3.split(y))
+
+    assert (
+        _make_splits_listlike(splits1)
+        == _make_splits_listlike(splits2)
+        == _make_splits_listlike(splits3)
+    )