Repository: statsmodels/statsmodels

Issue:
Extending SARIMAXResults ignores DatetimeIndex
#### Describe the bug
Extending SARIMAXResults ignores DatetimeIndex and throws the following warning:
```
/home/user/Programs/anaconda3/envs/vision/lib/python3.7/site-packages/statsmodels/tsa/base/tsa_model.py:219: ValueWarning: A date index has been provided, but it has no associated frequency information and so will be ignored when e.g. forecasting.
  ' ignored when e.g. forecasting.', ValueWarning)
```

#### Code Sample, a copy-pastable example if possible
data: [my_data_sample2.csv](https://drive.google.com/file/d/185CkaiFpeKsy978PnawqoclwVS3tIOzQ/view?usp=sharing)

```python
endog = pd.read_csv("my_data_sample2.csv", index_col=0, parse_dates=True)

n_sample = 500
endog_train = endog.iloc[:n_sample]
endog_test = endog.iloc[n_sample:n_sample*2]

mod = sm.tsa.SARIMAX(endog, order=(1, 0, 0), trend='c')
res = mod.fit(disp=False)

endog_new_point = endog_test[0:1 + 1]
res = res.extend(endog_new_point)
# res = res.append(endog_new_point, refit=False)
pred_res_all = res.get_forecast(steps=2)
pred_means = pred_res_all.predicted_mean

print(pred_means)
```
Output:

```
2    100.838670
3    100.838663
dtype: float64
```
If the issue has not been resolved, please file it in the issue tracker.

#### Expected Output
The output index is expected to be datetime, the same as indices produced by `append`.
```
2019-04-24 22:50:40    100.838670
2019-04-24 22:51:40    100.838663
Freq: T, dtype: float64
```

Code that generated this output:


```python
endog = pd.read_csv("my_data_sample2.csv", index_col=0, parse_dates=True)

n_sample = 500
endog_train = endog.iloc[:n_sample]
endog_test = endog.iloc[n_sample:n_sample*2]

mod = sm.tsa.SARIMAX(endog, order=(1, 0, 0), trend='c')
res = mod.fit(disp=False)

endog_new_point = endog_test[0:1 + 1]
# res = res.extend(endog_new_point)
res = res.append(endog_new_point, refit=False)
pred_res_all = res.get_forecast(steps=2)
pred_means = pred_res_all.predicted_mean

print(pred_means)
```

#### Output of ``import statsmodels.api as sm; sm.show_versions()``

`0.11.0.dev0+628.g17f7c3481`



Discussion:
Basically the issue here is that the date index doesn't have a frequency. The model infers the frequency to be minutes, but then the new dataset passed to `extend` is too short for Pandas to infer the frequency, so it just considers it to be an unsupported index.

I think the solution here will be to raise an exception if the new data passed to `extend` (or `append`) doesn't have an index that can be matched to the end of the model's index. At least then users will have clarity about the solution to the problem.

Resolving patch:
diff --git a/statsmodels/tsa/statespace/mlemodel.py b/statsmodels/tsa/statespace/mlemodel.py
--- a/statsmodels/tsa/statespace/mlemodel.py
+++ b/statsmodels/tsa/statespace/mlemodel.py
@@ -61,6 +61,26 @@ def _handle_args(names, defaults, *args, **kwargs):
     return tuple(output_args) + (kwargs,)
 
 
+def _check_index(desired_index, dta, title='data'):
+    given_index = None
+    if isinstance(dta, (pd.Series, pd.DataFrame)):
+        given_index = dta.index
+    if given_index is not None and not desired_index.equals(given_index):
+        desired_freq = getattr(desired_index, 'freq', None)
+        given_freq = getattr(given_index, 'freq', None)
+        if ((desired_freq is not None or given_freq is not None) and
+                desired_freq != given_freq):
+            raise ValueError('Given %s does not have an index'
+                             ' that extends the index of the'
+                             ' model. Expected index frequency is'
+                             ' "%s", but got "%s".'
+                             % (title, desired_freq, given_freq))
+        else:
+            raise ValueError('Given %s does not have an index'
+                             ' that extends the index of the'
+                             ' model.' % title)
+
+
 class MLEModel(tsbase.TimeSeriesModel):
     r"""
     State space model for maximum likelihood estimation
@@ -3607,16 +3627,37 @@ def append(self, endog, exog=None, refit=False, fit_kwargs=None, **kwargs):
         statsmodels.tsa.statespace.mlemodel.MLEResults.extend
         statsmodels.tsa.statespace.mlemodel.MLEResults.apply
         """
+        start = self.nobs
+        end = self.nobs + len(endog) - 1
+        _, _, _, append_ix = self.model._get_prediction_index(start, end)
+
+        # Check the index of the new data
+        if isinstance(self.model.data, PandasData):
+            _check_index(append_ix, endog, '`endog`')
+
+        # Concatenate the new data to original data
         new_endog = concat([self.model.data.orig_endog, endog], axis=0,
                            allow_mix=True)
+
+        # Create a continuous index for the combined data
         if isinstance(self.model.data, PandasData):
             start = 0
             end = len(new_endog) - 1
             _, _, _, new_index = self.model._get_prediction_index(start, end)
-            new_endog = pd.Series(new_endog, index=new_index)
+            # Standardize `endog` to have the right index and columns
+            columns = self.model.endog_names
+            if not isinstance(columns, list):
+                columns = [columns]
+            new_endog = pd.DataFrame(new_endog, index=new_index,
+                                     columns=columns)
+
+        # Handle `exog`
         if exog is not None:
             _, exog = prepare_exog(exog)
-            new_exog = concat([self.model.data.orig_exog, exog], axis=0)
+            _check_index(append_ix, exog, '`exog`')
+
+            new_exog = concat([self.model.data.orig_exog, exog], axis=0,
+                              allow_mix=True)
         else:
             new_exog = None
 
@@ -3700,6 +3741,18 @@ def extend(self, endog, exog=None, fit_kwargs=None, **kwargs):
         statsmodels.tsa.statespace.mlemodel.MLEResults.append
         statsmodels.tsa.statespace.mlemodel.MLEResults.apply
         """
+        start = self.nobs
+        end = self.nobs + len(endog) - 1
+        _, _, _, extend_ix = self.model._get_prediction_index(start, end)
+
+        if isinstance(self.model.data, PandasData):
+            _check_index(extend_ix, endog, '`endog`')
+
+            # Standardize `endog` to have the right index and columns
+            columns = self.model.endog_names
+            if not isinstance(columns, list):
+                columns = [columns]
+            endog = pd.DataFrame(endog, index=extend_ix, columns=columns)
         # Extend the current fit result to additional data
         mod = self.model.clone(endog, exog=exog, **kwargs)
         mod.ssm.initialization = Initialization(
diff --git a/statsmodels/tsa/statespace/tools.py b/statsmodels/tsa/statespace/tools.py
--- a/statsmodels/tsa/statespace/tools.py
+++ b/statsmodels/tsa/statespace/tools.py
@@ -306,6 +306,22 @@ def concat(series, axis=0, allow_mix=False):
     is_pandas = np.r_[[_is_using_pandas(s, None) for s in series]]
 
     if np.all(is_pandas):
+        if isinstance(series[0], pd.DataFrame):
+            base_columns = series[0].columns
+        else:
+            base_columns = pd.Index([series[0].name])
+        for s in series[1:]:
+            if isinstance(s, pd.DataFrame):
+                s_columns = s.columns
+            else:
+                s_columns = pd.Index([s.name])
+
+            if axis == 0 and not base_columns.equals(s_columns):
+                raise ValueError('Columns must match to concatenate along'
+                                 ' rows.')
+            elif axis == 1 and not series[0].index.equals(s.index):
+                raise ValueError('Index must match to concatenate along'
+                                 ' columns.')
         concatenated = pd.concat(series, axis=axis)
     elif np.all(~is_pandas) or allow_mix:
         concatenated = np.concatenate(series, axis=axis)


Test patch:
diff --git a/statsmodels/tsa/statespace/tests/test_mlemodel.py b/statsmodels/tsa/statespace/tests/test_mlemodel.py
--- a/statsmodels/tsa/statespace/tests/test_mlemodel.py
+++ b/statsmodels/tsa/statespace/tests/test_mlemodel.py
@@ -1040,3 +1040,62 @@ def test_append_extend_apply_invalid():
     assert_raises(ValueError, res1.append, endog2, fit_kwargs={'cov_kwds': {}})
     assert_raises(ValueError, res1.extend, endog2, fit_kwargs={'cov_kwds': {}})
     assert_raises(ValueError, res1.apply, endog2, fit_kwargs={'cov_kwds': {}})
+
+    # Test for exception when given a different frequency
+    wrong_freq = niledata.iloc[20:40]
+    wrong_freq.index = pd.date_range(
+        start=niledata.index[0], periods=len(wrong_freq), freq='MS')
+    message = ('Given `endog` does not have an index that extends the index of'
+               ' the model. Expected index frequency is')
+    with pytest.raises(ValueError, match=message):
+        res1.append(wrong_freq)
+    with pytest.raises(ValueError, match=message):
+        res1.extend(wrong_freq)
+    message = ('Given `exog` does not have an index that extends the index of'
+               ' the model. Expected index frequency is')
+    with pytest.raises(ValueError, match=message):
+        res1.append(endog2, exog=wrong_freq)
+    message = 'The indices for endog and exog are not aligned'
+    with pytest.raises(ValueError, match=message):
+        res1.extend(endog2, exog=wrong_freq)
+
+    # Test for exception when given the same frequency but not right after the
+    # end of model
+    not_cts = niledata.iloc[21:41]
+    message = ('Given `endog` does not have an index that extends the index of'
+               ' the model.$')
+    with pytest.raises(ValueError, match=message):
+        res1.append(not_cts)
+    with pytest.raises(ValueError, match=message):
+        res1.extend(not_cts)
+    message = ('Given `exog` does not have an index that extends the index of'
+               ' the model.$')
+    with pytest.raises(ValueError, match=message):
+        res1.append(endog2, exog=not_cts)
+    message = 'The indices for endog and exog are not aligned'
+    with pytest.raises(ValueError, match=message):
+        res1.extend(endog2, exog=not_cts)
+
+    # # Test for problems with non-date indexes
+    endog3 = pd.Series(niledata.iloc[:20].values)
+    endog4 = pd.Series(niledata.iloc[:40].values)[20:]
+    mod2 = sarimax.SARIMAX(endog3, order=(1, 0, 0), exog=endog3,
+                           concentrate_scale=True)
+    res2 = mod2.smooth([0.2, 0.5])
+
+    # Test for exception when given the same frequency but not right after the
+    # end of model
+    not_cts = pd.Series(niledata[:41].values)[21:]
+    message = ('Given `endog` does not have an index that extends the index of'
+               ' the model.$')
+    with pytest.raises(ValueError, match=message):
+        res2.append(not_cts)
+    with pytest.raises(ValueError, match=message):
+        res2.extend(not_cts)
+    message = ('Given `exog` does not have an index that extends the index of'
+               ' the model.$')
+    with pytest.raises(ValueError, match=message):
+        res2.append(endog4, exog=not_cts)
+    message = 'The indices for endog and exog are not aligned'
+    with pytest.raises(ValueError, match=message):
+        res2.extend(endog4, exog=not_cts)