Repository: statsmodels/statsmodels

Issue:
BUG : arima_model._get_predict_out_of_sample, ignores exogenous of there is no trend ?
Hello,

I have a feeling there is a problem with `arima_model._get_predict_out_of_sample()` if there is an exogenous variable, but no trend.  Looking at the code, it just doesn't use exog if there is no trend.

I have produced a sample script to reproduce the problem.

In the sample, arma_mod2.fit() produces a forecast of zero.  It produces a forecast of zero, whatever I set exog to be.  I believe the forecast should be the 'my_forecast' variable.

Apologies in advance if I have misread / misunderstood the documentation or how it is supposed to work.

Example code:

```
import numpy as np
import pandas as pd

import statsmodels.api as sm
from statsmodels.tsa.arima_process import arma_generate_sample

np.random.seed(12345)

arparams = np.array([.75, -.25])
maparams = np.array([.65, .35])

arparam = np.r_[1, -arparams]
maparam = np.r_[1, maparams]

nobs = 20

dates = sm.tsa.datetools.dates_from_range('1980',length=nobs)

y  = arma_generate_sample(arparams, maparams, nobs)

df       = pd.DataFrame({'y':y},index=dates)
df['y1'] = df['y'].shift(1)

arma_mod2     = sm.tsa.ARMA( df['y'][1:] , order=(0,0), exog=df['y1'][1:] )
arma_res2     = arma_mod2.fit(trend='nc', disp=False)    
arma_forecast = arma_res2.forecast(exog = df['y'][-1])

my_forecast   = df['y'][-1] * arma_res2.params[0]

print(' y[-1]            : {0:0.4f}'.format( df['y'][-1] ) )
print( 'Exog coef        : {0:0.4f}'.format( arma_res2.params[0] ) )
print( 'Model Forecast   : {0:0.4f}'.format( arma_forecast[0] ) )
print( 'My Forecast      : {0:0.4f}'.format( my_forecast ) )
```



Discussion:
What version are you using?

[**Edit**: nm. This problem exists in current master.]

I think there is a problem here but this is an odd case. I'm not sure ARMA should support an order (0,0) fit. Many changes would have to be made.


Resolving patch:
diff --git a/statsmodels/tsa/arima_model.py b/statsmodels/tsa/arima_model.py
--- a/statsmodels/tsa/arima_model.py
+++ b/statsmodels/tsa/arima_model.py
@@ -219,10 +219,14 @@ def _get_predict_out_of_sample(endog, p, q, k_trend, k_exog, start, errors,
             mu = trendparam * (1 - arparams.sum())
             # arparams were reversed in unpack for ease later
             mu = mu + (np.r_[1, -arparams[::-1]]*X).sum(1)[:,None]
-
         else:
             mu = trendparam * (1 - arparams.sum())
             mu = np.array([mu]*steps)
+    elif k_exog > 0:
+        X = np.dot(exog, exparams)
+        #NOTE: you shouldn't have to give in-sample exog!
+        X = lagmat(X, p, original='in', trim='both')
+        mu = (np.r_[1, -arparams[::-1]]*X).sum(1)[:,None]
     else:
         mu = np.zeros(steps)
 
@@ -1365,6 +1369,21 @@ def forecast(self, steps=1, exog=None, alpha=.05):
         conf_int : array
             2d array of the confidence interval for the forecast
         """
+        if exog is not None:
+            #TODO: make a convenience function for this. we're using the
+            # pattern elsewhere in the codebase
+            exog = np.asarray(exog)
+            if self.k_exog == 1 and exog.ndim == 1:
+                exog = exog[:,None]
+            elif exog.ndim == 1:
+                if len(exog) != self.k_exog:
+                    raise ValueError("1d exog given and len(exog) != k_exog")
+                exog = exog[None, :]
+            if exog.shape[0] != steps:
+                raise ValueError("new exog needed for each step")
+            # prepend in-sample exog observations
+            exog = np.vstack((self.model.exog[-self.k_ar:, self.k_trend:],
+                              exog))
 
         arparams = self.arparams
         maparams = self.maparams
@@ -1612,6 +1631,14 @@ def forecast(self, steps=1, exog=None, alpha=.05):
         Prediction is done in the levels of the original endogenous variable.
         If you would like prediction of differences in levels use `predict`.
         """
+        if exog is not None:
+            if self.k_exog == 1 and exog.ndim == 1:
+                exog = exog[:,None]
+            if exog.shape[0] != steps:
+                raise ValueError("new exog needed for each step")
+            # prepend in-sample exog observations
+            exog = np.vstack((self.model.exog[-self.k_ar:, self.k_trend:],
+                              exog))
         forecast = _arma_predict_out_of_sample(self.params, steps, self.resid,
                                         self.k_ar, self.k_ma, self.k_trend,
                                         self.k_exog, self.model.endog,


Test patch:
diff --git a/statsmodels/tsa/tests/test_arima.py b/statsmodels/tsa/tests/test_arima.py
--- a/statsmodels/tsa/tests/test_arima.py
+++ b/statsmodels/tsa/tests/test_arima.py
@@ -1673,7 +1673,7 @@ def test_arima_predict_exog():
     params = np.array([2.786912485145725, -0.122650190196475,
                        0.533223846028938, -0.319344321763337,
                        0.132883233000064])
-    assert_almost_equal(arma_res.params, params, 6)
+    assert_almost_equal(arma_res.params, params, 5)
     # no exog for in-sample
     predict = arma_res.predict()
     assert_almost_equal(predict, predict_expected.values[:100], 5)
@@ -1828,9 +1828,52 @@ def test_arima_exog_predict_1d():
     y = np.random.random(100)
     x = np.random.random(100)
     mod = ARMA(y, (2, 1), x).fit(disp=-1)
-    newx = np.random.random(12)
+    newx = np.random.random(10)
     results = mod.forecast(steps=10, alpha=0.05, exog=newx)
 
+def test_arima_1123():
+    # test ARMAX predict when trend is none
+    np.random.seed(12345)
+    arparams = np.array([.75, -.25])
+    maparams = np.array([.65, .35])
+
+    arparam = np.r_[1, -arparams]
+    maparam = np.r_[1, maparams]
+
+    nobs = 20
+
+    dates = dates_from_range('1980',length=nobs)
+
+    y = arma_generate_sample(arparams, maparams, nobs)
+
+    X = np.random.randn(nobs)
+    y += 5*X
+    mod = ARMA(y[:-1], order=(1,0), exog=X[:-1])
+    res = mod.fit(trend='nc', disp=False)
+    fc = res.forecast(exog=X[-1:])
+    # results from gretl
+    assert_almost_equal(fc[0], 2.200393, 6)
+    assert_almost_equal(fc[1], 1.030743, 6)
+    assert_almost_equal(fc[2][0,0], 0.180175, 6)
+    assert_almost_equal(fc[2][0,1], 4.220611, 6)
+
+    mod = ARMA(y[:-1], order=(1,1), exog=X[:-1])
+    res = mod.fit(trend='nc', disp=False)
+    fc = res.forecast(exog=X[-1:])
+    assert_almost_equal(fc[0], 2.765688, 6)
+    assert_almost_equal(fc[1], 0.835048, 6)
+    assert_almost_equal(fc[2][0,0], 1.129023, 6)
+    assert_almost_equal(fc[2][0,1], 4.402353, 6)
+
+    # make sure this works to. code looked fishy.
+    mod = ARMA(y[:-1], order=(1,0), exog=X[:-1])
+    res = mod.fit(trend='c', disp=False)
+    fc = res.forecast(exog=X[-1:])
+    assert_almost_equal(fc[0], 2.481219, 6)
+    assert_almost_equal(fc[1], 0.968759, 6)
+    assert_almost_equal(fc[2][0], [0.582485, 4.379952], 6)
+
+
 if __name__ == "__main__":
     import nose
     nose.runmodule(argv=[__file__, '-vvs', '-x', '--pdb'], exit=False)