Repository: sktime/sktime

Issue:
[BUG] convert to period bug in `_StatsModelsAdapter`
**Describe the bug**
<!--
A clear and concise description of what the bug is.
-->

For panel data with datetime indexes, converting datetimes to period index causes an error in the predict stage in the `_StatsModelsAdapter` models

**To Reproduce**
<!--
Add a Minimal, Complete, and Verifiable example (for more details, see e.g. https://stackoverflow.com/help/mcve

If the code is too long, feel free to put it in a public gist and link it in the issue: https://gist.github.com
-->

```python
from sktime.datasets import load_airline
from sktime.forecasting.exp_smoothing import ExponentialSmoothing
import pandas as pd
y = load_airline()

# create dummy index with hourly timestamps and panel data by hour of day
y.index = pd.date_range(start='1960-01-01', periods=len(y.index), freq='H')
y.index.names = ["datetime"]
y.name = "passengers"
y = y.to_frame()
y['hour_of_day'] = y.index.hour
y = y.reset_index().set_index(['hour_of_day', 'datetime']).sort_index()

forecaster = ExponentialSmoothing(
    trend='add', sp=1
)  
forecaster.fit(y)
forecaster.predict(fh=[1])

```

**Expected behavior**
<!--
A clear and concise description of what you expected to happen.
-->

For each panel predict one step ahead

**Additional context**
<!--
Add any other context about the problem here.
-->

I will address this issue 👍 

**Versions**
<details>

<!--
Please run the following code snippet and paste the output here:

from sktime import show_versions; show_versions()
-->

Python dependencies:
          pip: 23.1.2
       sktime: 0.24.1
      sklearn: 1.2.2
       skbase: 0.6.1
        numpy: 1.24.3
        scipy: 1.10.1
       pandas: 2.0.2
   matplotlib: None
       joblib: 1.2.0
        numba: None
  statsmodels: 0.14.0
     pmdarima: 2.0.3
statsforecast: None
      tsfresh: None
      tslearn: None
        torch: None
   tensorflow: None
tensorflow_probability: None

<!-- Thanks for contributing! -->



Discussion:

Resolving patch:
diff --git a/sktime/forecasting/base/adapters/_statsmodels.py b/sktime/forecasting/base/adapters/_statsmodels.py
--- a/sktime/forecasting/base/adapters/_statsmodels.py
+++ b/sktime/forecasting/base/adapters/_statsmodels.py
@@ -2,7 +2,7 @@
 # copyright: sktime developers, BSD-3-Clause License (see LICENSE file)
 """Implements adapter for statsmodels forecasters to be used in sktime framework."""
 
-__author__ = ["mloning"]
+__author__ = ["mloning", "ciaran-g"]
 __all__ = ["_StatsModelsAdapter"]
 
 import inspect
@@ -99,7 +99,7 @@ def _predict(self, fh, X):
         # statsmodels requires zero-based indexing starting at the
         # beginning of the training series when passing integers
         start, end = fh.to_absolute_int(self._y.index[0], self.cutoff)[[0, -1]]
-        fh_abs = fh.to_absolute_index(self.cutoff)
+        fh_int = fh.to_absolute_int(self._y.index[0], self.cutoff) - len(self._y)
 
         # bug fix for evaluate function as test_plus_train indices are passed
         # statsmodels exog must contain test indices only.
@@ -108,7 +108,7 @@ def _predict(self, fh, X):
             ind_drop = self._X.index
             X = X.loc[~X.index.isin(ind_drop)]
             # Entire range of the forecast horizon is required
-            X = X[: fh_abs[-1]]
+            X = X.iloc[: (fh_int[-1] + 1)]  # include end point
 
         if "exog" in inspect.signature(self._forecaster.__init__).parameters.keys():
             y_pred = self._fitted_forecaster.predict(start=start, end=end, exog=X)
@@ -117,7 +117,9 @@ def _predict(self, fh, X):
 
         # statsmodels forecasts all periods from start to end of forecasting
         # horizon, but only return given time points in forecasting horizon
-        y_pred = y_pred.loc[fh_abs]
+        # if fh[0] > 1 steps ahead of cutoff then make relative to `start`
+        fh_int = fh_int - fh_int[0]
+        y_pred = y_pred.iloc[fh_int]
         # ensure that name is not added nor removed
         # otherwise this may upset conversion to pd.DataFrame
         y_pred.name = self._y.name
@@ -186,7 +188,9 @@ def _predict_interval(self, fh, X, coverage):
             return BaseForecaster._predict_interval(self, fh, X=X, coverage=coverage)
 
         start, end = fh.to_absolute_int(self._y.index[0], self.cutoff)[[0, -1]]
-        valid_indices = fh.to_absolute(self.cutoff).to_pandas()
+        fh_int = fh.to_absolute_int(self._y.index[0], self.cutoff) - len(self._y)
+        # if fh > 1 steps ahead of cutoff
+        fh_int = fh_int - fh_int[0]
 
         get_prediction_arguments = {"start": start, "end": end}
 
@@ -205,17 +209,15 @@ def _predict_interval(self, fh, X, coverage):
         var_names = self._get_varnames()
         var_name = var_names[0]
         columns = pd.MultiIndex.from_product([var_names, coverage, ["lower", "upper"]])
-        pred_int = pd.DataFrame(index=valid_indices, columns=columns)
+        preds_index = self._extract_conf_int(prediction_results, (1 - coverage[0]))
+        preds_index = preds_index.iloc[fh_int].index
+        pred_int = pd.DataFrame(index=preds_index, columns=columns)
 
         for c in coverage:
             pred_statsmodels = self._extract_conf_int(prediction_results, (1 - c))
 
-            pred_int[(var_name, c, "lower")] = pred_statsmodels.loc[
-                valid_indices, "lower"
-            ]
-            pred_int[(var_name, c, "upper")] = pred_statsmodels.loc[
-                valid_indices, "upper"
-            ]
+            pred_int[(var_name, c, "lower")] = pred_statsmodels.iloc[fh_int]["lower"]
+            pred_int[(var_name, c, "upper")] = pred_statsmodels.iloc[fh_int]["upper"]
 
         return pred_int
 


Test patch:
diff --git a/sktime/forecasting/tests/test_exp_smoothing.py b/sktime/forecasting/tests/test_exp_smoothing.py
--- a/sktime/forecasting/tests/test_exp_smoothing.py
+++ b/sktime/forecasting/tests/test_exp_smoothing.py
@@ -1,11 +1,13 @@
 """Test exponential smoothing forecasters."""
 
-__author__ = ["mloning", "big-o"]
+__author__ = ["mloning", "big-o", "ciaran-g"]
 __all__ = ["test_set_params"]
 
+import pandas as pd
 import pytest
 from numpy.testing import assert_array_equal
 
+from sktime.datasets import load_airline
 from sktime.forecasting.exp_smoothing import ExponentialSmoothing
 from sktime.split import temporal_train_test_split
 from sktime.utils._testing.forecasting import make_forecasting_problem
@@ -35,3 +37,23 @@ def test_set_params():
     y_pred = f.predict()
 
     assert_array_equal(y_pred, expected)
+
+
+@pytest.mark.skipif(
+    not _check_soft_dependencies("statsmodels", severity="none"),
+    reason="skip test if required soft dependency not available",
+)
+def check_panel_expsmooth():
+    """Test exponential smoothing on panel data with datetime index."""
+    # make panel with hour of day panel and datetime index
+    y = load_airline()
+    y.index = pd.date_range(start="1960-01-01", periods=len(y.index), freq="H")
+    y.index.names = ["datetime"]
+    y.name = "passengers"
+    y = y.to_frame()
+    y["hour_of_day"] = y.index.hour
+    y = y.reset_index().set_index(["hour_of_day", "datetime"]).sort_index()
+
+    forecaster = ExponentialSmoothing(trend="add", sp=1)
+    forecaster.fit(y)
+    forecaster.predict(fh=[1, 3])
diff --git a/sktime/forecasting/tests/test_theta.py b/sktime/forecasting/tests/test_theta.py
--- a/sktime/forecasting/tests/test_theta.py
+++ b/sktime/forecasting/tests/test_theta.py
@@ -1,9 +1,10 @@
 """Tests for ThetaForecaster."""
 # copyright: sktime developers, BSD-3-Clause License (see LICENSE file)
 
-__author__ = ["big-o", "kejsitake"]
+__author__ = ["big-o", "kejsitake", "ciaran-g"]
 
 import numpy as np
+import pandas as pd
 import pytest
 
 from sktime.datasets import load_airline
@@ -121,3 +122,24 @@ def test_theta_and_thetamodular():
     y_pred_thetamodular = f1.predict(fh=fh)
 
     np.testing.assert_allclose(y_pred_theta, y_pred_thetamodular, rtol=0.06)
+
+
+@pytest.mark.skipif(
+    not _check_soft_dependencies("statsmodels", severity="none"),
+    reason="skip test if required soft dependency not available",
+)
+def check_panel_theta_quantiles():
+    """Test predict quantiles with theta on panel data with datetime index."""
+    # make panel with hour of day panel and datetime index
+    y = load_airline()
+    y.index = pd.date_range(start="1960-01-01", periods=len(y.index), freq="H")
+    y.index.names = ["datetime"]
+    y.name = "passengers"
+    y = y.to_frame()
+    y["hour_of_day"] = y.index.hour
+    y = y.reset_index().set_index(["hour_of_day", "datetime"]).sort_index()
+
+    forecaster = ThetaForecaster(sp=1)
+    forecaster.fit(y)
+    forecaster.predict(fh=[1, 3])
+    forecaster.predict_quantiles(fh=[1, 3], alpha=[0.1, 0.5, 0.9])