Repository: statsmodels/statsmodels

Issue:
Exogenous variable restarts from index 0 when crossing last in-sample date
When predicting over a period that includes in-sample dates followed by out-of-sample dates, `predict` goes back to the first index in the exogenous variable when crossing the last in-sample date.

Expected behavior: respect the time/dates in the exogenous variable; don't start from 0 in the middle of the prediction.

Reproducer (iPython Notebook):

```
%matplotlib inline
import matplotlib.pyplot as plt
import numpy
import numpy as np
import pandas
import statsmodels.api as sm
numpy.set_printoptions(linewidth=120)

fit_start_date = '2012-01-01'
fit_end_date = '2014-06-29' 
predict_end_date = '2014-12-28'

# Define an exogenous variables.
date_index = pandas.date_range(fit_start_date, predict_end_date, freq='W-SUN')
season_indicator = [abs(x.timetuple().tm_yday - 182) / 182.0 for x in date_index]
exog = pandas.TimeSeries(season_indicator, index=date_index)

# Define the endogenous variable as a scaled and noisy version of the same thing, up to fit_end_date.
np.random.seed(0)  # Keep noise the same between runs.
noise = pandas.Series(np.abs(np.random.randn(len(date_index))), index=date_index).div(3.0)
endog = pandas.Series(season_indicator, index=date_index).mul(2.0).pow(2.0).add(noise)[:fit_end_date]

arma = sm.tsa.ARMA(endog=endog, exog=exog[:fit_end_date],
                    order=(2, 0), freq='W')
model = arma.fit(disp=False)

# Predictions crossing over fit_end_date
ar_predictions = model.predict(
        start=fit_start_date,
        end=predict_end_date,
        exog=exog)

# A workaround to get expected values.
ar_predictions_workaround = model.predict(
        start=fit_end_date,
        end=predict_end_date,
        exog=exog[fit_end_date:predict_end_date])

# Print exog, endog, actual predictions, and workaround (expected) predictions.
plt.figure(figsize=(16,5))
plt.plot(endog.index, endog, 'g', label='Endogenous')
plt.plot(exog.index, exog, 'm', label='Exogenous')
plt.plot(ar_predictions.index, ar_predictions, 'rx-', label='Actual prediction')
plt.plot(ar_predictions_workaround.index, ar_predictions_workaround, 'kx-', label='Expected prediction')
plt.legend(loc='upper center')

print 'StatsModels version:', sm.version.full_version
print model.params
print model.mle_retvals
```

Output:

```
StatsModels version: 0.6.1
const     -0.405181
x1         4.142397
ar.L1.y    0.584564
ar.L2.y    0.198743
dtype: float64
{'gopt': array([  4.30211422e-08,   2.22044605e-08,  -4.30211422e-08,   0.00000000e+00]), 'fcalls': 78, 'converged': True, 'fopt': 0.08108206616357157, 'warnflag': 0}
```

Also see attached graph.
![screenshot from 2015-03-16 14 52 26](https://cloud.githubusercontent.com/assets/1418594/6668633/18a84c40-cbec-11e4-8f1c-388a7f237cab.png)



Discussion:
I don't know why this ticket got lost, without label and milestone.

It's likely that this was the ARMA predict with exog bug fixed for 0.7.
TODO: check if this is still broken

This is still broken.

Resolving patch:
diff --git a/statsmodels/tsa/arima_model.py b/statsmodels/tsa/arima_model.py
--- a/statsmodels/tsa/arima_model.py
+++ b/statsmodels/tsa/arima_model.py
@@ -7,6 +7,7 @@
 from datetime import datetime
 
 import numpy as np
+import pandas as pd
 from scipy import optimize
 from scipy.stats import t, norm
 from scipy.signal import lfilter
@@ -27,6 +28,7 @@
 from statsmodels.tsa.arima_process import arma2ma
 from statsmodels.tools.numdiff import approx_hess_cs, approx_fprime_cs
 from statsmodels.tools.validation import array_like
+from statsmodels.tools.sm_exceptions import SpecificationWarning
 from statsmodels.tsa.kalmanf import KalmanFilter
 
 _armax_notes = r"""
@@ -119,15 +121,16 @@
             want out of sample prediction.
         exog : array_like, optional
             If the model is an ARMAX and out-of-sample forecasting is
-            requested, exog must be given. Note that you'll need to pass
-            `k_ar` additional lags for any exogenous variables. E.g., if you
-            fit an ARMAX(2, q) model and want to predict 5 steps, you need 7
-            observations to do this.
+            requested, exog must be given. exog must be aligned so that exog[0]
+            is used to produce the first out-of-sample forecast. The number of
+            observation in exog should match the number of out-of-sample
+            forecasts produced. If the length of exog does not match the number
+            of forecasts, a SpecificationWarning is produced.
         dynamic : bool, optional
             The `dynamic` keyword affects in-sample prediction. If dynamic
             is False, then the in-sample lagged values are used for
             prediction. If `dynamic` is True, then in-sample forecasts are
-            used in place of lagged dependent variables. The first forecasted
+            used in place of lagged dependent variables. The first forecast
             value is `start`.
         %(extra_params)s
 
@@ -233,6 +236,37 @@ def cumsum_n(x, n):
     return x
 
 
+def _prediction_adjust_exog(exog, row_labels, dynamic, end):
+    """
+    Adjust exog if exog has dates that align with endog
+
+    Parameters
+    ----------
+    exog : {array_like, None}
+        The exog values
+    row_labels : {pd.DatetimeIndex, None}
+        Row labels from endog
+    dynamic : bool
+        Flag indicating whether dynamic forecasts are expected
+    end : int
+        Index of final in-sample observation
+    """
+    if exog is None:
+        return None
+
+    exog_start = 0
+    exog_index = getattr(exog, 'index', None)
+    exog_dates = isinstance(exog_index, pd.DatetimeIndex)
+    endog_dates = isinstance(row_labels, pd.DatetimeIndex)
+    date_adj = endog_dates and exog_dates and not dynamic
+    if date_adj and row_labels.isin(exog_index).all():
+        end_label = row_labels[end]
+        exog_start = exog.index.get_loc(end_label) + 1
+
+    exog = array_like(exog, 'exog', ndim=2)
+    return exog[exog_start:]
+
+
 def _check_arima_start(start, k_ar, k_diff, method, dynamic):
     if start < 0:
         raise ValueError("The start index %d of the original series "
@@ -701,7 +735,7 @@ def geterrors(self, params):
     def predict(self, params, start=None, end=None, exog=None, dynamic=False,
                 **kwargs):
         if kwargs and 'typ' not in kwargs:
-            raise TypeError('Unkown extra arguments')
+            raise TypeError('Unknown extra arguments')
         method = getattr(self, 'method', 'mle')  # do not assume fit
         # params = np.asarray(params)
 
@@ -716,12 +750,9 @@ def predict(self, params, start=None, end=None, exog=None, dynamic=False,
         resid = self.geterrors(params)
         k_ar = self.k_ar
 
-        if exog is not None:
-            # Note: we ignore currently the index of exog if it is available
-            exog = array_like(exog, 'exog', ndim=2)
-            if self.k_exog == 1 and exog.ndim == 1:
-                exog = exog[:, None]
-
+        # Adjust exog if exog has dates that align with endog
+        row_labels = self.data.row_labels
+        exog = _prediction_adjust_exog(exog, row_labels, dynamic, end)
         if out_of_sample != 0 and self.k_exog > 0:
             # we need the last k_ar exog for the lag-polynomial
             if self.k_exog > 0 and k_ar > 0 and not dynamic:
@@ -753,7 +784,24 @@ def predict(self, params, start=None, end=None, exog=None, dynamic=False,
                                                          self.k_ma,
                                                          self.k_trend,
                                                          self.k_exog, endog,
-                                                         exog, method=method)
+                                                         exog,
+                                                         method=method)
+            if (exog is not None and
+                    (exog.shape[0] - k_ar) != forecastvalues.shape[0]):
+
+                import warnings
+                msg = """
+The number of observations in exog does not match the number of out-of-sample
+observations.  This might indicate that exog is not correctly aligned. exog
+should be aligned so that the exog[0] is used for the first out-of-sample
+forecast, and exog[-1] is used for the last out-of-sample forecast.
+exog is not used for in-sample observations which are the fitted values.
+
+To silence this warning, ensure the number of observation in exog ({0})
+matches the number of out-of-sample forecasts ({1})'
+"""
+                msg = msg.format(exog.shape[0], forecastvalues.shape[0])
+                warnings.warn(msg, SpecificationWarning)
             predictedvalues = np.r_[predictedvalues, forecastvalues]
         return predictedvalues
     predict.__doc__ = _arma_predict
@@ -1519,7 +1567,8 @@ def forecast(self, steps=1, exog=None, alpha=.05):
         exog : array
             If the model is an ARMAX, you must provide out of sample
             values for the exogenous variables. This should not include
-            the constant.
+            the constant. The number of observation in exog must match the
+            value of steps.
         alpha : float
             The confidence intervals for the forecasts are (1 - alpha) %
 
@@ -1842,7 +1891,8 @@ def forecast(self, steps=1, exog=None, alpha=.05):
         exog : array
             If the model is an ARIMAX, you must provide out of sample
             values for the exogenous variables. This should not include
-            the constant.
+            the constant. The number of observation in exog must match the
+            value of steps.
         alpha : float
             The confidence intervals for the forecasts are (1 - alpha) %
 
@@ -1861,8 +1911,7 @@ def forecast(self, steps=1, exog=None, alpha=.05):
         If you would like prediction of differences in levels use `predict`.
         """
         if exog is not None:
-            if self.k_exog == 1 and exog.ndim == 1:
-                exog = exog[:, None]
+            exog = array_like(exog, 'exog', ndim=2)
             if exog.shape[0] != steps:
                 raise ValueError("new exog needed for each step")
             if self.k_exog != exog.shape[1]:
@@ -1883,7 +1932,7 @@ def forecast(self, steps=1, exog=None, alpha=.05):
                                                self.model.endog,
                                                exog, method=self.model.method)
 
-        d = self.k_diff
+        d = self.model.k_diff
         endog = self.model.data.endog[-d:]
         forecast = unintegrate(forecast, unintegrate_levels(endog, d))[d:]
 


Test patch:
diff --git a/statsmodels/tsa/tests/test_arima.py b/statsmodels/tsa/tests/test_arima.py
--- a/statsmodels/tsa/tests/test_arima.py
+++ b/statsmodels/tsa/tests/test_arima.py
@@ -17,7 +17,7 @@
 import statsmodels.sandbox.tsa.fftarma as fa
 from statsmodels.tools.testing import assert_equal
 from statsmodels.tools.sm_exceptions import (
-    ValueWarning, HessianInversionWarning)
+    ValueWarning, HessianInversionWarning, SpecificationWarning)
 from statsmodels.tsa.arma_mle import Arma
 from statsmodels.tsa.arima_model import AR, ARMA, ARIMA
 from statsmodels.regression.linear_model import OLS
@@ -2517,3 +2517,28 @@ def test_arima_d0_forecast(reset_randomstate):
     assert_allclose(pred, pred_typ)
     pred_mod = mod.predict(res.params, start=200, end=300, typ='linear')
     assert_allclose(pred, pred_mod)
+
+
+def test_predict_exog_adjust(reset_randomstate):
+    df = pd.DataFrame(np.random.standard_normal((1000, 2)),
+                      index=pd.date_range('31-12-2000', periods=1000),
+                      columns=['y', 'x'])
+    y = df.y.iloc[:750]
+    x = df.x.iloc[:750]
+    mod = ARMA(y, (2, 0), exog=x)
+    res = mod.fit(disp=-1)
+    pred_adj = res.predict(end=df.index[-1], exog=df.x)
+    pred = res.predict(end=df.index[-1], exog=df.x.iloc[750:])
+    assert_allclose(pred, pred_adj)
+
+
+def test_predict_wrong_exog_len(reset_randomstate):
+    df = pd.DataFrame(np.random.standard_normal((1000, 2)),
+                      index=pd.date_range('31-12-2000', periods=1000),
+                      columns=['y', 'x'])
+    y = df.y.iloc[:750]
+    x = df.x.iloc[:750]
+    mod = ARMA(y, (2, 0), exog=x)
+    res = mod.fit(disp=-1)
+    with pytest.warns(SpecificationWarning, match='of observations in exog'):
+        res.predict(end=df.index[-1], exog=df.x.iloc[500:])