Repository: sktime/sktime
Pull request #6437 (2024-05-17 11:00:12)

[ENH] Interval width (sharpness) metric
Interval width metric for interval forecasts, as requeste by @benHeid.


Patch:
diff --git a/docs/source/api_reference/forecasting.rst b/docs/source/api_reference/forecasting.rst
index bfd68ecf0..97cc002fe 100644
--- a/docs/source/api_reference/forecasting.rst
+++ b/docs/source/api_reference/forecasting.rst
@@ -364,6 +364,8 @@ Deep learning based forecasters
     LTSFNLinearForecaster
 
 .. currentmodule:: sktime.forecasting.hf_transformers_forecaster
+
+.. autosummary::
     :toctree: auto_generated/
     :template: class.rst
 
diff --git a/docs/source/api_reference/split.rst b/docs/source/api_reference/split.rst
index ee1b27fb7..742ce6c0b 100644
--- a/docs/source/api_reference/split.rst
+++ b/docs/source/api_reference/split.rst
@@ -47,6 +47,7 @@ They have tag ``"split_type"="temporal"``.
     SingleWindowSplitter
     SlidingWindowSplitter
     ExpandingWindowSplitter
+    ExpandingCutoffSplitter
     ExpandingGreedySplitter
     TemporalTrainTestSplitter
 
diff --git a/docs/source/api_reference/transformations.rst b/docs/source/api_reference/transformations.rst
index d07f454ae..1bedaa6f2 100644
--- a/docs/source/api_reference/transformations.rst
+++ b/docs/source/api_reference/transformations.rst
@@ -137,6 +137,14 @@ These transformers extract simple summary features.
     RandomIntervalFeatureExtractor
     FittedParamExtractor
 
+.. currentmodule:: sktime.transformations.series.adi_cv
+
+.. autosummary::
+    :toctree: auto_generated/
+    :template: class.rst
+
+    ADICVTransformer
+
 Shapelets, wavelets, and convolution
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
diff --git a/docs/source/developer_guide/deprecation.rst b/docs/source/developer_guide/deprecation.rst
index 74eabe412..4d0db2c4f 100644
--- a/docs/source/developer_guide/deprecation.rst
+++ b/docs/source/developer_guide/deprecation.rst
@@ -4,13 +4,44 @@
 Deprecation
 ===========
 
-Before we can make changes to sktime's user interface, we need to make sure that users have time to make the necessary adjustments in their code.
-For this reason, we first need to deprecate functionality and change it only in a next release.
+``sktime`` aims to be stable and reliable towards its users.
+Our high-level policy to ensure this is:
 
-.. note::
+"``sktime`` should never break user code without a clear and actionable warning
+given at least one (MINOR) release cycle in advance."
+
+Here, "break" expressly includes a change to abstract logic, such as the algorithm
+being used, not just changes that lead to exceptions or performance degradation.
+
+For instance, if a user has code
+
+.. code:: python
+
+    from sktime.forecasting.foo import BarForecaster
+
+    bar = BarForecaster(42, x=43)
+    bar.fit(y_train, fh=[1, 2, 3])
+    y_pred = bar.predict()
+
+then no release of ``sktime`` should change, without warning:
+
+* import location of ``BarForecaster``
+* argument signature of ``BarForecaster``, including name, order, and defaults of arguments
+* the abstract algorithm that ``BarForecaster`` carries out for the given arguments
+
+Changes that can be carried out without warning:
+
+* adding more arguments at the end of the argument list, with a default value that retains prior behaviour,
+  as long as the new arguments are well-documented
+* pure refactoring of internal code, as long as the public API remains the same
+* changing the implementation without changing the abstract algorithm, e.g., for performance reasons
+
+The deprecation policy outlined in this document provides details on how to carry out
+changes that need change or deprecation handling, in a user-friendly and reliable way.
+
+It is accompanied by formulaic patterns for developers, with examples,
+and a process for release managers, to make the policy easy to follow.
 
-    For upcoming changes and next releases, see our `Milestones <https://github.com/sktime/sktime/milestones?direction=asc&sort=due_date&state=open>`_.
-    For our long-term plan, see our :ref:`roadmap`.
 
 Deprecation policy
 ==================
diff --git a/pyproject.toml b/pyproject.toml
index 2e57e0231..d8f6dd31e 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -207,7 +207,7 @@ regression = [
 transformations = [
   'esig<0.10,>=0.9.7; python_version < "3.11"',
   "filterpy<1.5,>=1.4.5",
-  "holidays>=0.29,<0.49",
+  "holidays>=0.29,<0.50",
   "mne>=1.5,<1.8",
   'numba<0.60,>=0.53',
   "pycatch22>=0.4,<0.4.6",
@@ -273,7 +273,7 @@ dl = [
   'neuralforecast<1.8.0,>=1.6.4; python_version < "3.11"',
   'tensorflow<2.17,>=2; python_version < "3.12"',
   'torch; python_version < "3.12"',
-  'transformers[torch]; python_version < "3.12"',
+  'transformers[torch]<4.41.0; python_version < "3.12"',
 ]
 mlflow = [
   "mlflow",
diff --git a/sktime/datatypes/_adapter/dask_to_pd.py b/sktime/datatypes/_adapter/dask_to_pd.py
index 7e07fa458..aa6bc5bb5 100644
--- a/sktime/datatypes/_adapter/dask_to_pd.py
+++ b/sktime/datatypes/_adapter/dask_to_pd.py
@@ -44,11 +44,11 @@ def convert_dask_to_pandas(obj):
 
     Parameters
     ----------
-    obj : pandas.DataFrame
+    obj : dask DataFrame
 
     Returns
     -------
-    dask DataFrame
+    pandas.DataFrame
         MultiIndex levels 0 .. -1 of X are converted to columns of name
         __index__[indexname], where indexname is name of multiindex level,
         or the integer index if the level has no name
@@ -86,7 +86,7 @@ def convert_pandas_to_dask(obj, npartitions=1, chunksize=None, sort=True):
 
     Parameters
     ----------
-    obj : dask DataFrame
+    obj : pandas.DataFrame
     npartitions : int or None, optional, default = 1
         npartitions passed to dask from_pandas when converting obj to dask
     chunksize : int or None, optional, default = None
@@ -96,7 +96,7 @@ def convert_pandas_to_dask(obj, npartitions=1, chunksize=None, sort=True):
 
     Returns
     -------
-    pandas.DataFrame
+    dask DataFrame
         MultiIndex levels 0 .. -1 of X are converted to columns of name
         __index__[indexname], where indexname is name of multiindex level,
         or the integer index if the level has no name
diff --git a/sktime/networks/base.py b/sktime/networks/base.py
index 7d04a18fb..4cf14fff7 100644
--- a/sktime/networks/base.py
+++ b/sktime/networks/base.py
@@ -4,15 +4,7 @@ __author__ = ["Withington", "TonyBagnall"]
 
 from abc import ABC, abstractmethod
 
-import numpy as np
-import pandas as pd
-
 from sktime.base import BaseObject
-from sktime.forecasting.base import BaseForecaster
-from sktime.utils.validation._dependencies import _check_soft_dependencies
-
-if _check_soft_dependencies("torch", severity="none"):
-    import torch
 
 
 class BaseDeepNetwork(BaseObject, ABC):
@@ -35,244 +27,3 @@ class BaseDeepNetwork(BaseObject, ABC):
         output_layer : a keras layer
         """
         ...
-
-
-class BaseDeepNetworkPyTorch(BaseForecaster, ABC):
-    """Abstract base class for deep learning networks using torch.nn."""
-
-    _tags = {
-        "python_dependencies": "torch",
-        "y_inner_mtype": "pd.DataFrame",
-        "capability:insample": False,
-        "capability:pred_int:insample": False,
-        "scitype:y": "both",
-        "ignores-exogeneous-X": True,
-    }
-
-    def __init__(
-        self,
-        num_epochs=16,
-        batch_size=8,
-        in_channels=1,
-        individual=False,
-        criterion_kwargs=None,
-        optimizer=None,
-        optimizer_kwargs=None,
-        lr=0.001,
-    ):
-        self.num_epochs = num_epochs
-        self.batch_size = batch_size
-        self.in_channels = in_channels
-        self.individual = individual
-        self.criterion_kwargs = criterion_kwargs
-        self.optimizer = optimizer
-        self.optimizer_kwargs = optimizer_kwargs
-        self.lr = lr
-
-        super().__init__()
-
-    def _fit(self, y, fh, X=None):
-        """Fit the network.
-
-        Changes to state:
-            writes to self._network.state_dict
-
-        Parameters
-        ----------
-        X : iterable-style or map-style dataset
-            see (https://pytorch.org/docs/stable/data.html) for more information
-        """
-        from sktime.forecasting.base import ForecastingHorizon
-
-        # save fh and y for prediction later
-        if fh.is_relative:
-            self._fh = fh
-        else:
-            fh = fh.to_relative(self.cutoff)
-            self._fh = fh
-
-        self._y = y
-
-        if type(fh) is ForecastingHorizon:
-            self.network = self._build_network(fh._values[-1])
-        else:
-            self.network = self._build_network(fh)
-
-        if self.criterion:
-            if self.criterion in self.criterions.keys():
-                if self.criterion_kwargs:
-                    self._criterion = self.criterions[self.criterion](
-                        **self.criterion_kwargs
-                    )
-                else:
-                    self._criterion = self.criterions[self.criterion]()
-            else:
-                raise TypeError(
-                    f"Please pass one of {self.criterions.keys()} for `criterion`."
-                )
-        else:
-            # default criterion
-            self._criterion = torch.nn.MSELoss()
-
-        if self.optimizer:
-            if self.optimizer in self.optimizers.keys():
-                if self.optimizer_kwargs:
-                    self._optimizer = self.optimizers[self.optimizer](
-                        self.network.parameters(), lr=self.lr, **self.optimizer_kwargs
-                    )
-                else:
-                    self._optimizer = self.optimizers[self.optimizer](
-                        self.network.parameters(), lr=self.lr
-                    )
-            else:
-                raise TypeError(
-                    f"Please pass one of {self.optimizers.keys()} for `optimizer`."
-                )
-        else:
-            # default optimizer
-            self._optimizer = torch.optim.Adam(self.network.parameters(), lr=self.lr)
-
-        dataloader = self.build_pytorch_train_dataloader(y)
-        self.network.train()
-
-        for _ in range(self.num_epochs):
-            for x, y in dataloader:
-                y_pred = self.network(x)
-                loss = self._criterion(y_pred, y)
-                self._optimizer.zero_grad()
-                loss.backward()
-                self._optimizer.step()
-
-    def _predict(self, X=None, fh=None):
-        """Predict with fitted model."""
-        from torch import cat
-
-        if fh is None:
-            fh = self._fh
-
-        if max(fh._values) > self.network.pred_len or min(fh._values) < 0:
-            raise ValueError(
-                f"fh of {fh} passed to {self.__class__.__name__} is not "
-                "within `pred_len`. Please use a fh that aligns with the `pred_len` of "
-                "the forecaster."
-            )
-
-        if X is None:
-            dataloader = self.build_pytorch_pred_dataloader(self._y, fh)
-        else:
-            dataloader = self.build_pytorch_pred_dataloader(X, fh)
-
-        y_pred = []
-        for x, _ in dataloader:
-            y_pred.append(self.network(x).detach())
-        y_pred = cat(y_pred, dim=0).view(-1, y_pred[0].shape[-1]).numpy()
-        y_pred = y_pred[fh._values.values - 1]
-        y_pred = pd.DataFrame(
-            y_pred, columns=self._y.columns, index=fh.to_absolute_index(self.cutoff)
-        )
-
-        return y_pred
-
-    def build_pytorch_train_dataloader(self, y):
-        """Build PyTorch DataLoader for training."""
-        from torch.utils.data import DataLoader
-
-        if self.custom_dataset_train:
-            if hasattr(self.custom_dataset_train, "build_dataset") and callable(
-                self.custom_dataset_train.build_dataset
-            ):
-                self.custom_dataset_train.build_dataset(y)
-                dataset = self.custom_dataset_train
-            else:
-                raise NotImplementedError(
-                    "Custom Dataset `build_dataset` method is not available. Please "
-                    f"refer to the {self.__class__.__name__}.build_dataset "
-                    "documentation."
-                )
-        else:
-            dataset = PyTorchTrainDataset(
-                y=y,
-                seq_len=self.network.seq_len,
-                fh=self._fh._values[-1],
-            )
-
-        return DataLoader(
-            dataset,
-            self.batch_size,
-        )
-
-    def build_pytorch_pred_dataloader(self, y, fh):
-        """Build PyTorch DataLoader for prediction."""
-        from torch.utils.data import DataLoader
-
-        if self.custom_dataset_pred:
-            if hasattr(self.custom_dataset_pred, "build_dataset") and callable(
-                self.custom_dataset_pred.build_dataset
-            ):
-                self.custom_dataset_train.build_dataset(y)
-                dataset = self.custom_dataset_train
-            else:
-                raise NotImplementedError(
-                    "Custom Dataset `build_dataset` method is not available. Please"
-                    f"refer to the {self.__class__.__name__}.build_dataset"
-                    "documentation."
-                )
-        else:
-            dataset = PyTorchPredDataset(
-                y=y[-self.network.seq_len :],
-                seq_len=self.network.seq_len,
-            )
-
-        return DataLoader(
-            dataset,
-            self.batch_size,
-        )
-
-    def get_y_true(self, y):
-        """Get y_true values for validation."""
-        dataloader = self.build_pytorch_pred_dataloader(y)
-        y_true = [y.flatten().numpy() for _, y in dataloader]
-        return np.concatenate(y_true, axis=0)
-
-
-class PyTorchTrainDataset:
-    """Dataset for use in sktime deep learning forecasters."""
-
-    def __init__(self, y, seq_len, fh):
-        self.y = y.values
-        self.seq_len = seq_len
-        self.fh = fh
-
-    def __len__(self):
-        """Return length of dataset."""
-        return len(self.y) - self.seq_len - self.fh + 1
-
-    def __getitem__(self, i):
-        """Return data point."""
-        from torch import from_numpy, tensor
-
-        return (
-            tensor(self.y[i : i + self.seq_len]).float(),
-            from_numpy(self.y[i + self.seq_len : i + self.seq_len + self.fh]).float(),
-        )
-
-
-class PyTorchPredDataset:
-    """Dataset for use in sktime deep learning forecasters."""
-
-    def __init__(self, y, seq_len):
-        self.y = y.values
-        self.seq_len = seq_len
-
-    def __len__(self):
-        """Return length of dataset."""
-        return 1
-
-    def __getitem__(self, i):
-        """Return data point."""
-        from torch import from_numpy, tensor
-
-        return (
-            tensor(self.y[i : i + self.seq_len]).float(),
-            from_numpy(self.y[i + self.seq_len : i + self.seq_len]).float(),
-        )
diff --git a/sktime/performance_metrics/forecasting/_coerce.py b/sktime/performance_metrics/forecasting/_coerce.py
index ee191f2b7..da61c7c89 100644
--- a/sktime/performance_metrics/forecasting/_coerce.py
+++ b/sktime/performance_metrics/forecasting/_coerce.py
@@ -18,6 +18,8 @@ def _coerce_to_scalar(obj):
 
 def _coerce_to_df(obj):
     """Coerce to pd.DataFrame, from polymorphic input scalar or pandas."""
+    if isinstance(obj, (float, int)):
+        return pd.DataFrame([obj])
     return pd.DataFrame(obj)
 
 
diff --git a/sktime/performance_metrics/forecasting/probabilistic/__init__.py b/sktime/performance_metrics/forecasting/probabilistic/__init__.py
index 5d17073e0..fbab9f8df 100644
--- a/sktime/performance_metrics/forecasting/probabilistic/__init__.py
+++ b/sktime/performance_metrics/forecasting/probabilistic/__init__.py
@@ -17,6 +17,7 @@ __all__ = [
     "EmpiricalCoverage",
     "ConstraintViolation",
     "CRPS",
+    "IntervalWidth",
     "LogLoss",
     "SquaredDistrLoss",
 ]
@@ -25,6 +26,7 @@ from sktime.performance_metrics.forecasting.probabilistic._classes import (
     CRPS,
     ConstraintViolation,
     EmpiricalCoverage,
+    IntervalWidth,
     LogLoss,
     PinballLoss,
     SquaredDistrLoss,
diff --git a/sktime/performance_metrics/forecasting/probabilistic/_classes.py b/sktime/performance_metrics/forecasting/probabilistic/_classes.py
index 42d727677..cfee5b76f 100644
--- a/sktime/performance_metrics/forecasting/probabilistic/_classes.py
+++ b/sktime/performance_metrics/forecasting/probabilistic/_classes.py
@@ -599,6 +599,70 @@ class EmpiricalCoverage(_BaseProbaForecastingErrorMetric):
         return [params1]
 
 
+class IntervalWidth(_BaseProbaForecastingErrorMetric):
+    """Interval width for interval predictions, sometimes also known as sharpness.
+
+    Parameters
+    ----------
+    multioutput : string "uniform_average" or "raw_values" determines how\
+        multioutput results will be treated.
+
+    score_average : bool, optional, default = True
+        specifies whether scores for each quantile should be averaged.
+    """
+
+    _tags = {
+        "scitype:y_pred": "pred_interval",
+        "lower_is_better": True,
+    }
+
+    def __init__(self, multioutput="uniform_average", score_average=True):
+        self.score_average = score_average
+        self.multioutput = multioutput
+        super().__init__(score_average=score_average, multioutput=multioutput)
+
+    def _evaluate_by_index(self, y_true, y_pred, multioutput, **kwargs):
+        """Logic for finding the metric evaluated at each index.
+
+        y_true : pd.Series, pd.DataFrame or np.array of shape (fh,) or \
+            (fh, n_outputs) where fh is the forecasting horizon
+            Ground truth (correct) target values.
+
+        y_pred : pd.Series, pd.DataFrame or np.array of shape (fh,) or  \
+            (fh, n_outputs)  where fh is the forecasting horizon
+            Forecasted values.
+
+        multioutput : string "uniform_average" or "raw_values" determines how \
+            multioutput results will be treated.
+        """
+        lower = y_pred.iloc[:, y_pred.columns.get_level_values(2) == "lower"].to_numpy()
+        upper = y_pred.iloc[:, y_pred.columns.get_level_values(2) == "upper"].to_numpy()
+
+        if not isinstance(y_true, np.ndarray):
+            y_true_np = y_true.to_numpy()
+        else:
+            y_true_np = y_true
+        if y_true_np.ndim == 1:
+            y_true_np = y_true.reshape(-1, 1)
+
+        scores = np.unique(np.round(y_pred.columns.get_level_values(1), 7))
+        vars = np.unique(y_pred.columns.get_level_values(0))
+
+        metric_array = upper - lower
+
+        out_df = pd.DataFrame(
+            metric_array, columns=pd.MultiIndex.from_product([vars, scores])
+        )
+
+        return out_df
+
+    @classmethod
+    def get_test_params(self):
+        """Retrieve test parameters."""
+        params1 = {}
+        return [params1]
+
+
 class ConstraintViolation(_BaseProbaForecastingErrorMetric):
     """Percentage of interval constraint violations for interval predictions.
 
diff --git a/sktime/split/__init__.py b/sktime/split/__init__.py
index 012fc952a..6c1c2e630 100644
--- a/sktime/split/__init__.py
+++ b/sktime/split/__init__.py
@@ -3,6 +3,7 @@
 __all__ = [
     "CutoffSplitter",
     "CutoffFhSplitter",
+    "ExpandingCutoffSplitter",
     "ExpandingGreedySplitter",
     "ExpandingWindowSplitter",
     "InstanceSplitter",
@@ -15,6 +16,7 @@ __all__ = [
 ]
 
 from sktime.split.cutoff import CutoffFhSplitter, CutoffSplitter
+from sktime.split.expandingcutoff import ExpandingCutoffSplitter
 from sktime.split.expandinggreedy import ExpandingGreedySplitter
 from sktime.split.expandingwindow import ExpandingWindowSplitter
 from sktime.split.instance import InstanceSplitter
diff --git a/sktime/split/expandingcutoff.py b/sktime/split/expandingcutoff.py
new file mode 100644
index 000000000..c4ca15d9a
--- /dev/null
+++ b/sktime/split/expandingcutoff.py
@@ -0,0 +1,213 @@
+#!/usr/bin/env python3 -u
+# copyright: sktime developers, BSD-3-Clause License (see LICENSE file)
+"""Splitter that successively expands the training window."""
+
+__author__ = ["ninedigits"]
+
+__all__ = [
+    "ExpandingCutoffSplitter",
+]
+
+from typing import Optional
+
+import numpy as np
+import pandas as pd
+
+from sktime.split.base import BaseSplitter
+from sktime.split.base._common import ACCEPTED_Y_TYPES, _check_fh
+from sktime.utils.validation.forecasting import check_step_length
+
+
+def _is_datetimelike(x):
+    return isinstance(x, (pd.Timestamp, np.datetime64, pd.Period))
+
+
+def _is_int(arg):
+    return isinstance(arg, (int, np.int_))
+
+
+def _validate_cutoff(cutoff):
+    if not (_is_datetimelike(cutoff) or _is_int(cutoff)):
+        raise TypeError(
+            "Cutoff value must be datetime-like or integer but instead "
+            f"found type(cutoff) = {type(cutoff)}"
+        )
+    return cutoff
+
+
+class ExpandingCutoffSplitter(BaseSplitter):
+    """
+    Expanding cutoff splitter for time series data.
+
+    This splitter combines elements of `ExpandingWindowSplitter` and `CutoffSplitter`
+    to create training and testing sets. Unlike `ExpandingWindowSplitter` which begins
+    with a fixed initial window, this splitter uses a specific cutoff point as the
+    starting window for the training set. The training set then expands incrementally
+    in each split until it reaches the end of the series.
+
+    The test set is defined by a forecast horizon relative to the last point in the
+    training set, containing as many subsequent indices as specified by the `fh`
+    parameter.
+
+    The valid types of y-index and cutoff pairings are datelike-datelike, datelike-int,
+    and int-int. When a datelike index is combined with an int cutoff, the cutoff
+    functions as an iloc indexer. When an int index is paired with a positive int
+    cutoff, the cutoff serves as a loc indexer. If the int cutoff is negative, it
+    functions as an iloc indexer.
+                          c
+                          |
+    |---------------------|----fh----|------|
+    | * * * * * * * * * * x x x x x x - - - |
+    | * * * * * * * * * * * x x x x x x - - |
+    | * * * * * * * * * * * * x x x x x x - |
+    | * * * * * * * * * * * * * x x x x x x |
+
+    ``c`` = cutoff date or index.
+
+    ``*`` = training fold.
+
+    ``x`` = test fold.
+
+    Parameters
+    ----------
+    cutoff (int or pd.Timestamp):
+        The initial cutoff point in the series, which marks the beginning of the
+        first test set.
+    fh (int, list, or np.array):
+        Forecasting horizon, determining the size and  indices of the test sets.
+        It can be an integer, a list, or an array.
+    step_length (int):
+        The step length to expand the training set size in each split.
+
+    Examples
+    --------
+    >>> import pandas as pd
+    >>> from sktime.split import ExpandingCutoffSplitter
+    >>> date_range = pd.date_range(start='2020-Q1', end='2021-Q3', freq='QS')
+    >>> y = pd.DataFrame(index=pd.PeriodIndex(date_range, freq='Q'))
+    >>> cutoff = pd.Period('2021-Q1')
+    >>> cv = ExpandingCutoffSplitter(cutoff=cutoff, fh=[1, 2], step_length=1)
+    >>> list(cv.split(y)) # doctest: +SKIP
+    [(array([0, 1, 2, 3]), array([4, 5])), (array([0, 1, 2, 3, 4]), array([5, 6]))]
+    """
+
+    def __init__(self, cutoff, fh, step_length):
+        super().__init__()
+        self.cutoff = _validate_cutoff(cutoff)
+        self.fh = fh
+        self._fh = _check_fh(fh)
+        self.step_length = step_length
+        return
+
+    def _split(self, y, fh=None):
+        """
+        Generate indices to split data into training and testing sets.
+
+        Parameters
+        ----------
+        y : array-like, shape = [n_samples]
+            Time series data.
+        fh : int, default=None
+            Forecast horizon, if None, uses self.fh
+
+        Yields
+        ------
+        train : ndarray
+            The training set indices for that split.
+        test : ndarray
+            The testing set indices for that split.
+        """
+        if fh is None:
+            fh = self._fh
+        offset = fh.to_numpy().max() + 1
+        for cutoff in self.get_cutoffs(y):
+            train_window = np.arange(0, cutoff + 1, step=1)
+            test_window = np.arange(cutoff + 1, cutoff + offset, step=1)
+            yield train_window, test_window
+
+    def _validate_y(self, y):
+        y = self._coerce_to_index(y)
+        cutoff = self.cutoff
+        y0 = y[0]
+        cond1 = _is_datetimelike(y0) and (_is_datetimelike(cutoff) or _is_int(cutoff))
+        cond2 = _is_int(y0) and _is_int(cutoff)
+        if not (cond1 or cond2):
+            raise TypeError(
+                "Valid combinations for y and cutoff types are "
+                "datelike-datelike, datelike-int, or int-int, but instead "
+                f"found {type(y0)}-{type(cutoff)}"
+            )
+        return y
+
+    def _get_first_cutoff_index(self, y_index):
+        y0 = y_index[0]
+        if _is_int(self.cutoff) and self.cutoff < 0:
+            index = len(y_index) + self.cutoff - 1
+            if index < 0:
+                raise IndexError(
+                    f"Cutoff of value {self.cutoff} is out of bounds "
+                    f"for y of length {len(y_index)}"
+                )
+        elif _is_datetimelike(y0) and _is_int(self.cutoff) and (self.cutoff > 0):
+            index = self.cutoff - 1
+        else:
+            index = np.argmax(y_index == self.cutoff) - 1
+            if index == -1:
+                raise TypeError(
+                    "Could not find matching index, make sure that "
+                    f"type(y) {type(y0)} is compatible with type(cutoff) "
+                    f"{type(self.cutoff)}"
+                )
+        return index
+
+    def get_cutoffs(self, y: Optional[ACCEPTED_Y_TYPES] = None) -> np.ndarray:
+        """Return the cutoff points in .iloc[] context.
+
+        Parameters
+        ----------
+        y : pd.Series or pd.Index, optional (default=None)
+            Time series to split
+
+        Returns
+        -------
+        cutoffs : 1D np.ndarray of int
+            iloc location indices, in reference to y, of cutoff indices
+        """
+        if y is None:
+            raise ValueError(
+                f"{self.__class__.__name__} requires `y` to compute the cutoffs."
+            )
+        y = self._validate_y(y)
+        fh = self._fh
+        step_length = check_step_length(self.step_length)
+        cutoff_index = self._get_first_cutoff_index(y)
+        cutoffs = np.array([cutoff_index])
+        offset = fh.to_numpy().max()
+        while cutoff_index + offset < len(y) - 1:
+            cutoff_index += step_length
+            cutoffs = np.append(cutoffs, cutoff_index)
+        return cutoffs
+
+    @classmethod
+    def get_test_params(cls, parameter_set="default"):
+        """Return testing parameter settings for the splitter.
+
+        Parameters
+        ----------
+        parameter_set : str, default="default"
+            Name of the set of test parameters to return, for use in tests. If no
+            special parameters are defined for a value, will return ``"default"`` set.
+
+        Returns
+        -------
+        params : dict or list of dict, default = {}
+            Parameters to create testing instances of the class
+            Each dict are parameters to construct an "interesting" test instance, i.e.,
+            ``MyClass(**params)`` or ``MyClass(**params[i])`` creates a valid test
+            instance.
+            ``create_test_instance`` uses the first (or only) dictionary in ``params``
+        """
+        return [
+            {"fh": [2, 4], "cutoff": 3, "step_length": 1},
+            {"fh": [1, 2, 3], "cutoff": 3, "step_length": 2},
+        ]
diff --git a/sktime/transformations/series/adi_cv.py b/sktime/transformations/series/adi_cv.py
new file mode 100644
index 000000000..f4dd919de
--- /dev/null
+++ b/sktime/transformations/series/adi_cv.py
@@ -0,0 +1,237 @@
+# copyright: sktime developers, BSD-3-Clause License (see LICENSE file)
+"""Feature transformer that returns features of time series including categories."""
+
+import pandas as pd
+
+from sktime.transformations.base import BaseTransformer
+
+__author__ = ["shlok191"]
+
+
+class ADICVTransformer(BaseTransformer):
+    r"""Transformer categorizing series into ADI-CV2 classes after Syntetos/Boylan.
+
+    Transforms a time series into a category label, which is one of:
+    ``"smooth"``, ``"erratic"``, ``"intermittent"``, or ``"lumpy"``.
+
+    The labels are based on the Average Demand Interval (ADI) and the Coefficient of
+    Variation (CV2) of the time series, using simple thresholding.
+    Optionally, the transformer can return the ADI and CV2 values as well.
+
+    Let :math:`x_t` be the value of the time series at times :math:`t = 1, 2, \dots, T`,
+    and let :math:`N` be the number of non-zero values in the time series, i.e.,
+    :math:`N = \text{card}\{t: x_t \neq 0\}`.
+
+    The ADI and CV2 are calculated as follows:
+
+    1. Average Demand Interval (ADI): The average time period between
+      time periods with non-zero demands, mathematically defined as:
+
+    .. math:: ADI = \frac{T}{N - 1}
+
+    2. Variance (CV2): Variance calculated on non-zero values
+    in the time series. Mathematical definition is:
+
+    .. math:: CV2 = \frac{1}{N}\sum_{t=1}^{T} x_t^2 - \left(\frac{1}{N}\sum_{t=1}^{T} x_t\right)^2  # noqa: E501
+
+    3. Class: Classification of time series on basis of ADI threshold
+    and CV2 threshold.
+
+    For thresholds ``ADI_threshold`` and ``CV2_threshold``, the classes are:
+
+    1. Smooth: If ``ADI <= adi_threshold`` and ``CV2 <= cv2_threshold``
+    2. Erratic: If ``ADI <= adi_threshold`` and ``CV2 > cv2_threshold``
+    3. Intermittent: If ``ADI > adi_threshold`` and ``cv2 <= cv2_threshold``
+    4. Lumpy: if ``ADI > adi_threshold`` and ``CV2 > cv2_threshold``
+
+    Default values for the thresholds are taken from the paper by Syntetos/Boylan [1].
+    namely, ``adi_threshold = 1.32`` and ``cv2_threshold = 0.49``.
+    They can also be adjusted by passing them as parameters to the transformer.
+
+    Parameters
+    ----------
+    adi_threshold : float (default = 1.32)
+        Specifies the ADI threshold utilized for classifying the time series
+
+    cv2_threshold : float (default = 0.49)
+        Specifies the CV2 threshold utilized for classifying the time series
+
+    features : list[str] | None (default = ['adi', 'cv2', 'class'])
+        Specifies all of the feature values to be calculated
+
+    Examples
+    --------
+    >>> from sktime.transformations.series.adi_cv import ADICVTransformer
+    >>> from sktime.datasets import load_airline
+    >>> y = load_airline()
+    >>> transformer = ADICVTransformer()
+    >>> y_hat = transformer.fit_transform(y)
+
+    References
+    ----------
+    [1]: John E. Boylan, Aris Syntetos: The Accuracy of Intermittent
+    Demand Estimates. International Journal of Forecasting, 1 Apr. 2005
+    """
+
+    _tags = {
+        "scitype:transform-input": "Series",
+        "scitype:transform-output": "Primitives",
+        "scitype:instancewise": False,
+        "scitype:transform-labels": "None",
+        "X_inner_mtype": "pd.DataFrame",
+        "y_inner_mtype": "None",
+        "univariate-only": True,  # Demand being the only variable
+        "requires_y": False,
+        "fit_is_empty": True,
+        "capability:inverse_transform": False,
+        "capability:unequal_length": False,
+        "handles-missing-data": False,
+        "authors": ["shlok191"],
+        "maintainers": ["shlok191"],
+    }
+
+    def __init__(self, features=None, adi_threshold=1.32, cv_threshold=0.49):
+        """Initialize the transformer and processes any provided parameters.
+
+        Parameters
+        ----------
+        features : List[str] | None, optional
+            List of features to compute. Defaults to None (all features)
+
+        adi_threshold : float, optional
+            Threshold for Average Demand Interval. Defaults to 1.32.
+
+        cv_threshold : float, optional
+            Threshold for Variance. Defaults to 0.49.
+
+        Raises
+        ------
+            ValueError: If features is provided and does not
+            contain 'adi','cv2', or 'class'.
+        """
+        self.adi_threshold = adi_threshold
+        self.cv_threshold = cv_threshold
+        self.features = features
+
+        self.features_internal = features
+
+        # Initialize the parent class
+        super().__init__()
+
+        # Checking if the features parameter is valid
+        if features is not None:
+            if not all(feature in ["adi", "cv2", "class"] for feature in features):
+                raise ValueError(
+                    "Error in ADICVTransformer: Invalid features list provided. "
+                    "The features argument must either be None, or a list of str "
+                    "containing one or multiple of the valid feature strings "
+                    "('adi', 'cv2', 'class')."
+                )
+
+        else:
+            # Helpful to transform None to default list for transform function
+            self.features_internal = ["adi", "cv2", "class"]
+
+    def _transform(self, X, y=None):
+        """Transform X and return a transformed version.
+
+        private _transform containing core logic, called from transform
+
+        Parameters
+        ----------
+        X : Series
+            Series of time series data this transform function aims to classify
+
+        y : Series | None, default=None
+            Not required for the classification of the time series
+
+        Returns
+        -------
+        X_transformed : pd.DataFrame
+
+            The returned DataFrame consists of the columns in the features list passed
+            during initialization. Specifically, the columns include (by default):
+
+                1. Average Demand Interval (ADI)
+                2. Variance (CV2)
+                3. categorical class
+        """
+        X_non_zero = X.to_numpy().nonzero()
+        X_non_zero = X.iloc[X_non_zero]
+
+        # Calculating ADI value based on formula from paper
+        adi_value = len(X) / (len(X_non_zero) - 1)
+
+        # Calculating variance for all non-zero values
+        variance = X_non_zero.var().iloc[0]
+        cv2_value = variance / len(X_non_zero)
+
+        # Calculating the class type
+
+        adi_low = adi_value <= self.adi_threshold
+        cv2_low = cv2_value <= self.cv_threshold
+        adi_high = not adi_low
+        cv2_high = not cv2_low
+
+        if adi_low and cv2_low:
+            class_type = "smooth"
+
+        elif adi_low and cv2_high:
+            class_type = "erratic"
+
+        elif adi_high and cv2_low:
+            class_type = "intermittent"
+
+        elif adi_high and cv2_high:
+            class_type = "lumpy"
+
+        # Collecting all values together into dict and converting to DF
+        return_dict = {}
+
+        if "adi" in self.features_internal:
+            return_dict["adi"] = [adi_value]
+
+        if "cv2" in self.features_internal:
+            return_dict["cv2"] = [cv2_value]
+
+        if "class" in self.features_internal:
+            return_dict["class"] = [class_type]
+
+        df = pd.DataFrame(return_dict)
+
+        # Ordering the dataframe in the correct order
+        df = df.loc[:, self.features_internal]
+
+        return df
+
+    # todo: return default parameters, so that a test instance can be created
+    #   required for automated unit and integration testing of estimator
+    @classmethod
+    def get_test_params(cls, parameter_set="default"):
+        """Return testing parameter settings for the estimator.
+
+        Parameters
+        ----------
+        parameter_set : str, default="default"
+            Name of the set of test parameters to return, for use in tests.
+            If no special parameters are defined for a value, will return
+            `"default"` set.
+
+        Returns
+        -------
+        params : dict or list of dict, default = {}
+            Parameters to create testing instances of the class
+            Each dict are parameters to construct an "interesting" test case
+        """
+        # Testing with 0 thresholds for both thresholds
+        # in independent test cases!
+
+        params1 = {}
+        params2 = {"features": None, "adi_threshold": 1.32, "cv_threshold": 0.49}
+        params3 = {
+            "features": ["adi", "class"],
+            "adi_threshold": 1.5,
+            "cv_threshold": 0.2,
+        }
+
+        return [params1, params2, params3]
diff --git a/sktime/utils/plotting.py b/sktime/utils/plotting.py
index 534a26fa5..fd2667c4d 100644
--- a/sktime/utils/plotting.py
+++ b/sktime/utils/plotting.py
@@ -505,22 +505,46 @@ def plot_windows(cv, y, title="", ax=None):
 
 
 def plot_calibration(y_true, y_pred, ax=None):
-    """Plot the calibration of a probabilistic forecast.
+    r"""Plot the calibration curve for a sample of quantile predictions.
 
-    Calculates internally the calibration of the quantile forecast and
-    visualise it.
+    Visualizes calibration of the quantile predictions.
 
-    x-axis: interval from 0 to 1
-    y-axis: interval from 0 to 1
-    plot elements: the calibration fo the forecast (blue) and the ideal
-        calibration (orange)
+    Computes the following calibration plot:
+
+    Let :math:`p_1, \dots, p_k` be the quantile points at which
+    predictions in ``y_pred`` were queried,
+    e.g., via ``alpha`` in ``predict_quantiles``.
+
+    Let :math:`y_1, \dots, y_N` be the actual values in ``y_true``,
+    and let :math:`\widehat{y}_{i,j}`, for `i = 1 \dots N, j = 1 \dota k`
+    be quantile predictions at quantile point :math:`p_j`,
+    of the conditional distribution of :math:`y_i`, as contained in ``y_pred``.
+
+    We compute the calibration indicators :math:`c_{i, j},`
+    as :math:`c_{i, j} = 1, \{ if } y_i \le \widehat{y}_{i,j} \text{ and } 0, \text{otherwise},`  # noqa: E501
+    and calibration fractions as
+
+    .. math:: \widehat{p}_j = \frac{1}{N} \sum_{i = 1}^N c_{i, j}.
+
+    If the quantile predictions are well-calibrated, we expect :math:`\widehat{p}_j`
+    to be close to :math:`p_j`.
+
+    x-axis: interval from 0 to 1, quantile points
+    y-axis: interval from 0 to 1, calibration fractions
+    plot elements: calibration curve of the quantile predictions (blue) and the ideal
+        calibration curve (orange), the curve with equation y = x.
+        Calibration curve are points :math:`(p_i, \widehat{p}_i), i = 1 \dots, k`;
+        Ideal curve is the curve with equation y = x,
+        containing points :math:`(p_i, p_i)`.
 
     Parameters
     ----------
     y_true : pd.Series, single columned pd.DataFrame, or single columned np.array.
-        The actual values of the forecast
+        The actual values
     y_pred : pd.DataFrame
-        The quantile forecast.
+        The quantile predictions,
+        formatted as returned by ``BaseDistribution.quantile``,
+        or ``predict_quantiles``
     ax : matplotlib.axes.Axes, optional (default=None)
         Axes on which to plot. If None, axes will be created and returned.
 


Test patch:
diff --git a/sktime/performance_metrics/forecasting/probabilistic/tests/test_probabilistic_metrics.py b/sktime/performance_metrics/forecasting/probabilistic/tests/test_probabilistic_metrics.py
index 2c683a688..b5e450d51 100644
--- a/sktime/performance_metrics/forecasting/probabilistic/tests/test_probabilistic_metrics.py
+++ b/sktime/performance_metrics/forecasting/probabilistic/tests/test_probabilistic_metrics.py
@@ -8,6 +8,7 @@ import pytest
 from sktime.performance_metrics.forecasting.probabilistic import (
     ConstraintViolation,
     EmpiricalCoverage,
+    IntervalWidth,
     PinballLoss,
 )
 from sktime.split import temporal_train_test_split
@@ -20,8 +21,9 @@ quantile_metrics = [
 ]
 
 interval_metrics = [
-    EmpiricalCoverage,
     ConstraintViolation,
+    EmpiricalCoverage,
+    IntervalWidth,
 ]
 
 all_metrics = interval_metrics + quantile_metrics
diff --git a/sktime/performance_metrics/forecasting/tests/test_metrics.py b/sktime/performance_metrics/forecasting/tests/test_metrics.py
index 2a21fc076..cb46d90c3 100644
--- a/sktime/performance_metrics/forecasting/tests/test_metrics.py
+++ b/sktime/performance_metrics/forecasting/tests/test_metrics.py
@@ -144,3 +144,19 @@ def test_make_scorer_sklearn():
     scorer = make_forecasting_scorer(mean_absolute_error, name="MAE")
 
     scorer.evaluate(pd.Series([1, 2, 3]), pd.Series([1, 2, 4]))
+
+
+def test_metric_coercion_bug():
+    """Tests for sensible output when using hierarchical arg with non-hierarchical data.
+
+    Failure case in bug #6413.
+    """
+    from sktime.performance_metrics.forecasting import MeanAbsoluteError
+
+    y_true = np.array([[0.5, 1], [-1, 1], [7, -6]])
+    y_pred = np.array([[0, 2], [-1, 2], [8, -5]])
+    mae = MeanAbsoluteError(multilevel="raw_values", multioutput=[0.4, 0.6])
+    metric = mae(y_true, y_pred)
+
+    assert isinstance(metric, pd.DataFrame)
+    assert metric.shape == (1, 1)
diff --git a/sktime/split/tests/test_expandingcutoff.py b/sktime/split/tests/test_expandingcutoff.py
new file mode 100644
index 000000000..7c7133120
--- /dev/null
+++ b/sktime/split/tests/test_expandingcutoff.py
@@ -0,0 +1,213 @@
+# copyright: sktime developers, BSD-3-Clause License (see LICENSE file)
+"""Tests for expanding cutoff splitter."""
+
+import numpy as np
+import pandas as pd
+import pytest
+
+from sktime.forecasting.base import ForecastingHorizon
+from sktime.split import ExpandingCutoffSplitter
+from sktime.split.tests.test_split import _check_cv
+from sktime.utils._testing.hierarchical import _make_hierarchical
+from sktime.utils._testing.series import _make_series
+
+
+def test_expandingcutoff_datelike_index_001():
+    """Test datetime index with _check_cv"""
+    y = _make_series(n_timepoints=10, random_state=42)
+    cutoff = y.index[3]
+    fh = ForecastingHorizon([1, 2, 3], freq=y.index.freq)
+    cv = ExpandingCutoffSplitter(cutoff=cutoff, fh=fh, step_length=1)
+    train_windows, test_windows, cutoffs, n_splits = _check_cv(cv, y)
+    np.testing.assert_array_equal(cutoffs, cv.get_cutoffs(y))
+
+
+def test_expandingcutoff_int_index_002():
+    """Test int index with _check_cv"""
+    y = _make_series(n_timepoints=10, index_type="int", random_state=42)
+    cutoff = y.index[3]
+    fh = ForecastingHorizon([1, 2, 3])
+    cv = ExpandingCutoffSplitter(cutoff=cutoff, fh=fh, step_length=1)
+    train_windows, test_windows, cutoffs, n_splits = _check_cv(cv, y)
+    np.testing.assert_array_equal(cutoffs, cv.get_cutoffs(y))
+
+
+def test_expandingcutoff_ytype_cutofftype_combos_003a():
+    """Test invalid param combo"""
+    # Datetime cutoff
+    i = 3
+    y1 = _make_series(n_timepoints=10, index_type="datetime", random_state=42)
+    fh = ForecastingHorizon([1, 2, 3])
+    cutoffs = [-7, y1.index[i], 5]
+    step_lengths = [1, 2, 3]
+    for cutoff in cutoffs:
+        for step_length in step_lengths:
+            cv1 = ExpandingCutoffSplitter(cutoff=cutoff, fh=fh, step_length=step_length)
+            _check_cv(cv1, y1)
+
+
+def test_expandingcutoff_ytype_cutofftype_combos_003b():
+    """Test invalid param combo"""
+    # Datetime cutoff
+    y1 = _make_series(n_timepoints=10, index_type="int", random_state=42)
+    fh = ForecastingHorizon([1, 2, 3])
+    cutoffs = [-7, y1.index[3]]
+    step_lengths = [1, 2, 3]
+    for cutoff in cutoffs:
+        for step_length in step_lengths:
+            cv1 = ExpandingCutoffSplitter(cutoff=cutoff, fh=fh, step_length=step_length)
+            _check_cv(cv1, y1)
+
+
+def test_expandingcutoff_ytype_cutofftype_combos_003c():
+    """Test invalid param combo"""
+    # Datetime cutoff
+    y1 = _make_series(n_timepoints=10, index_type="int", random_state=42)
+    fh = ForecastingHorizon([1, 2, 3])
+    # Y-index is int but cutoff is datetime
+    with pytest.raises(TypeError):
+        cv1 = ExpandingCutoffSplitter(
+            cutoff=pd.Timestamp("2000-01-01"), fh=fh, step_length=1
+        )
+        _check_cv(cv1, y1)
+
+
+def test_expandingcutoff_splitloc_004():
+    """Test split loc"""
+    y = _make_series(n_timepoints=10, random_state=42)
+    cutoff = y.index[3]
+    fh = ForecastingHorizon([1, 2, 3], freq=y.index.freq)
+    cv = ExpandingCutoffSplitter(cutoff=cutoff, fh=fh, step_length=1)
+    train_test_values = list(cv.split_loc(y))
+    last_train, last_test = train_test_values[-1]
+    expected_last_train = y.index[0:7]
+    expected_last_test = y.index[7:]
+    test_cases = [(expected_last_train, last_train), (expected_last_test, last_test)]
+    for expected, actual in test_cases:
+        np.testing.assert_array_equal(expected.values, actual.values)
+
+
+def test_expandingcutoff_hiearchical_splitloc_005():
+    """Test hiearchical splitloc with datetime"""
+    y = _make_hierarchical(
+        min_timepoints=6,
+        max_timepoints=10,
+        same_cutoff=True,
+        hierarchy_levels=(1, 2),
+        index_type="datetime",
+        random_state=42,
+    )
+    y_index = y.index.get_level_values(-1)
+    cutoff_index = -4
+    cutoff = y_index[cutoff_index]
+    fh = ForecastingHorizon([1, 2, 3])
+    cv = ExpandingCutoffSplitter(cutoff=cutoff, fh=fh, step_length=1)
+
+    train_test = list(cv.split_loc(y))
+
+    y_train_expected = (
+        y.groupby(level=-2)
+        .apply(lambda row: row[0:cutoff_index])
+        .index.get_level_values(-1)
+    )
+    y_test_expected = (
+        y.groupby(level=-2)
+        .apply(lambda row: row[cutoff_index : cutoff_index + fh[-1]])
+        .index.get_level_values(-1)
+    )
+    pd.testing.assert_index_equal(
+        y_train_expected, train_test[0][0].get_level_values(-1)
+    )
+    pd.testing.assert_index_equal(
+        y_test_expected, train_test[0][1].get_level_values(-1)
+    )
+
+
+def test_expandingcutoff_hiearchical_forecastbylevel_006():
+    """Test hiearchical with forecast by level"""
+    from sktime.forecasting.compose import ForecastByLevel
+    from sktime.forecasting.model_selection import ForecastingGridSearchCV
+    from sktime.forecasting.naive import NaiveForecaster
+    from sktime.utils._testing.hierarchical import _make_hierarchical
+
+    y = _make_hierarchical(
+        min_timepoints=6,
+        max_timepoints=10,
+        same_cutoff=True,
+        hierarchy_levels=(1, 2),
+        index_type="datetime",
+        random_state=42,
+    )
+    y_index = y.index.get_level_values(-1)
+    last_date = y_index.max()
+    cutoff_index = -4
+    cutoff = y_index[cutoff_index]
+    fh = ForecastingHorizon([1, 2, 3])
+    cv = ExpandingCutoffSplitter(cutoff=cutoff, fh=fh, step_length=1)
+    forecaster = NaiveForecaster()
+    param_grid = {"strategy": ["last", "mean", "drift"]}
+    gscv = ForecastByLevel(
+        ForecastingGridSearchCV(forecaster=forecaster, param_grid=param_grid, cv=cv)
+    )
+    gscv.fit(y, fh=fh)
+    y_pred = gscv.predict()
+    expected_last_forecast_date = last_date + pd.Timedelta(days=fh[-1])
+    actual_last_forecast_date = y_pred.index.max()
+    assert expected_last_forecast_date, actual_last_forecast_date
+
+
+def test_expandingcutoff_fh_list_007():
+    """Test fh as list with _check_cv"""
+    y = _make_series(n_timepoints=10, random_state=42)
+    cutoff = y.index[3]
+    fh = [1, 2, 3]
+    cv = ExpandingCutoffSplitter(cutoff=cutoff, fh=fh, step_length=1)
+    train_windows, test_windows, cutoffs, n_splits = _check_cv(cv, y)
+    np.testing.assert_array_equal(cutoffs, cv.get_cutoffs(y))
+
+
+def test_expanding_cutoff_period_008():
+    date_range = pd.date_range(
+        start=pd.Timestamp("2020-Q1"), end=pd.Timestamp("2021-Q4"), freq="QS"
+    )
+    y = pd.DataFrame({"date": date_range})
+    y = y.reset_index()
+    y = y.set_index("date")
+    y.index = pd.PeriodIndex(y.index, freq="Q")
+
+    cutoff = pd.Period("2021-Q1")
+    fh = ForecastingHorizon([1, 2, 3])
+    cv = ExpandingCutoffSplitter(cutoff=cutoff, fh=fh, step_length=2)
+    _check_cv(cv, y)
+
+    # pandas periods and timestamps are both datelike but don't work directly
+    cutoff = pd.Timestamp("2021-Q1")
+    with pytest.raises(TypeError):
+        cv = ExpandingCutoffSplitter(cutoff=cutoff, fh=fh, step_length=1)
+        _check_cv(cv, y)
+
+
+def _make_splits_listlike(splits):
+    splits_new = []
+    for train, test in splits:
+        splits_new.append([train.tolist(), test.tolist()])
+    return splits_new
+
+
+def test_expanding_cutoff_docstring_examples():
+    date_range = pd.date_range(start="2020-Q1", end="2021-Q3", freq="QS")
+    y = pd.DataFrame(index=pd.PeriodIndex(date_range, freq="Q"))
+    cv1 = ExpandingCutoffSplitter(cutoff=pd.Period("2021-Q1"), fh=[1, 2], step_length=1)
+    splits1 = list(cv1.split(y))
+
+    cv2 = ExpandingCutoffSplitter(cutoff=4, fh=[1, 2], step_length=1)
+    splits2 = list(cv2.split(y))
+
+    cv3 = ExpandingCutoffSplitter(cutoff=-3, fh=[1, 2], step_length=1)
+    splits3 = list(cv3.split(y))
+
+    assert (
+        _make_splits_listlike(splits1)
+        == _make_splits_listlike(splits2)
+        == _make_splits_listlike(splits3)
+    )
diff --git a/sktime/tests/tests/test_test_utils.py b/sktime/tests/tests/test_test_utils.py
index 5426678cf..c4f19470b 100644
--- a/sktime/tests/tests/test_test_utils.py
+++ b/sktime/tests/tests/test_test_utils.py
@@ -75,12 +75,12 @@ def test_run_test_for_class():
         assert reason_nodep == "False_no_change"
 
     # now check estimator with soft deps
-    run_nodep = run_test_for_class(f_with_deps)
+    run_wdep = run_test_for_class(f_with_deps)
     assert isinstance(run, bool)
 
     dep_present = _check_estimator_deps(f_with_deps, severity="none")
     if not dep_present:
-        assert not run_nodep
+        assert not run_wdep
 
     res = run_test_for_class(f_with_deps, return_reason=True)
     assert isinstance(res, tuple)
diff --git a/sktime/transformations/series/tests/test_adi_cv.py b/sktime/transformations/series/tests/test_adi_cv.py
new file mode 100644
index 000000000..143c9a73f
--- /dev/null
+++ b/sktime/transformations/series/tests/test_adi_cv.py
@@ -0,0 +1,132 @@
+# copyright: sktime developers, BSD-3-Clause License (see LICENSE file)
+"""Tests for adi_cv transformers for time series Series."""
+
+import numpy as np
+import pandas as pd
+import pytest
+
+from sktime.transformations.series.adi_cv import ADICVTransformer
+
+
+def _generate_smooth_series(size: int = 750):
+    """Generates a demand time series of the "smooth" category.
+
+    Parameters
+    ----------
+    size : int, optional
+        The size of the generated time series, by default 750
+
+    Returns
+    -------
+    Pandas.Series
+        Returns the generated series in the Pandas Series format.
+    """
+
+    # Generating a smooth series, we keep variance low by keeping the
+    # standard deviation to 0.25. Denoted as scale according to numpy docs
+
+    smooth_series = np.random.normal(loc=10, scale=0.25, size=size)
+
+    return pd.Series(smooth_series)
+
+
+def _generate_erratic_series(size: int = 750):
+    """Generates a demand time series of the "erratic" category.
+
+    Parameters
+    ----------
+    size : int, optional
+        The size of the generated time series, by default 750
+
+    Returns
+    -------
+    Pandas.Series
+        Returns the generated series in the Pandas Series format.
+    """
+
+    # Generating an erratic series, we keep variance high by keeping the
+    # standard deviation to 1 and then squaring the values
+
+    erratic_series = np.random.normal(loc=10, scale=2.5, size=size) ** 2
+
+    return pd.Series(erratic_series)
+
+
+def _generate_intermittent_series(size: int = 750):
+    """Generates a demand time series of the "intermittent" category.
+
+    Parameters
+    ----------
+    size : int, optional
+        The size of the generated time series, by default 750
+
+    Returns
+    -------
+    Pandas.Series
+        Returns the generated series in the Pandas Series format.
+    """
+
+    # Generating an intermittent series, we keep ADI high by
+    # setting only 10% of all values to non-zero values
+
+    intermittent_series = np.zeros(shape=(size,))
+    non_zero_indices = np.random.choice(size, size=size // 10, replace=False)
+
+    intermittent_series[non_zero_indices] = np.random.normal(10, 0.25, size=size // 10)
+
+    return pd.Series(intermittent_series)
+
+
+def _generate_lumpy_series(size: int = 750):
+    """Generates a demand time series of the "lumpy" category.
+
+    Parameters
+    ----------
+    size : int, optional
+        The size of the generated time series, by default 750
+
+    Returns
+    -------
+    Pandas.Series
+        Returns the generated series in the Pandas Series format.
+    """
+
+    # Generating a lumpy series, we keep ADI high by
+    # setting only 10% of all values to non-zero values
+
+    lumpy_series = np.zeros(shape=(size,))
+    non_zero_indices = np.random.choice(size, size=size // 10, replace=False)
+
+    lumpy_series[non_zero_indices] = np.random.normal(10, 2.5, size=size // 10) ** 2
+
+    return pd.Series(lumpy_series)
+
+
+# Defining all of the categories we wish to run tests for
+@pytest.mark.parametrize(
+    "series_generator, expected_class",
+    [
+        (_generate_smooth_series, "smooth"),
+        (_generate_erratic_series, "erratic"),
+        (_generate_intermittent_series, "intermittent"),
+        (_generate_lumpy_series, "lumpy"),
+    ],
+)
+def test_adi_cv_extractor(series_generator, expected_class):
+    """
+    Runs a PyTest for all 4 demand time series categories.
+
+    Parameters
+    ----------
+    series_generator
+        A function that generates a time series in the Pandas Series format.
+
+    expected_class
+        The expected class to be predicted by the transformer.
+    """
+
+    series = series_generator()
+    transformer = ADICVTransformer()
+
+    df = transformer.fit_transform(series)
+    assert df["class"].iloc[0] == expected_class