Repository: statsmodels/statsmodels

Issue:
ARIMA.predict does not pass exog in super.
See SO question [here](https://stackoverflow.com/questions/27464703/arima-exogenous-variable-out-of-sample). It's only for this combination of arguments.

ARIMA.predict does not pass exog in super.
See SO question [here](https://stackoverflow.com/questions/27464703/arima-exogenous-variable-out-of-sample). It's only for this combination of arguments.

BUG: ARMAX dynamic insample predict with exog - wrong exog
I'm not able to replicate dynamic prediction with exog from Stata.

My impression is that the exog slice is wrong in dynamic prediction, or I still doen't understand this.

ARMA with dynamic=False agrees with Stata at several decimals



Discussion:

Resolving patch:
diff --git a/statsmodels/tsa/arima_model.py b/statsmodels/tsa/arima_model.py
--- a/statsmodels/tsa/arima_model.py
+++ b/statsmodels/tsa/arima_model.py
@@ -351,7 +351,7 @@ def _arma_predict_out_of_sample(params, steps, errors, p, q, k_trend, k_exog,
         endog[i+p] = fcast
 
     #need to do one more without updating endog
-    forecast[-1] = mu[-1] + np.dot(arparams, endog[steps - 1:])
+    forecast[steps - 1] = mu[steps - 1] + np.dot(arparams, endog[steps - 1:])
     return forecast
 
 
@@ -718,14 +718,18 @@ def predict(self, params, start=None, end=None, exog=None, dynamic=False):
         k_ar = self.k_ar
 
         if out_of_sample != 0 and self.k_exog > 0:
+            exog = np.asarray(exog)
             if self.k_exog == 1 and exog.ndim == 1:
                 exog = exog[:, None]
                 # we need the last k_ar exog for the lag-polynomial
-            if self.k_exog > 0 and k_ar > 0:
+            if self.k_exog > 0 and k_ar > 0 and not dynamic:
                 # need the last k_ar exog for the lag-polynomial
                 exog = np.vstack((self.exog[-k_ar:, self.k_trend:], exog))
 
         if dynamic:
+            if self.k_exog > 0:
+                # need the last k_ar exog for the lag-polynomial
+                exog = np.vstack((self.exog[start - k_ar:, self.k_trend:], exog))
             #TODO: now that predict does dynamic in-sample it should
             # also return error estimates and confidence intervals
             # but how? len(endog) is not tot_obs
@@ -1131,7 +1135,7 @@ def predict(self, params, start=None, end=None, exog=None, typ='linear',
         elif typ == 'levels':
             endog = self.data.endog
             if not dynamic:
-                predict = super(ARIMA, self).predict(params, start, end,
+                predict = super(ARIMA, self).predict(params, start, end, exog,
                                                      dynamic)
 
                 start = self._get_predict_start(start, dynamic)


Test patch:
diff --git a/statsmodels/tsa/tests/test_arima.py b/statsmodels/tsa/tests/test_arima.py
--- a/statsmodels/tsa/tests/test_arima.py
+++ b/statsmodels/tsa/tests/test_arima.py
@@ -1,7 +1,7 @@
 from statsmodels.compat.python import lrange, BytesIO
 import numpy as np
 from nose.tools import nottest
-from numpy.testing import (assert_almost_equal, assert_,
+from numpy.testing import (assert_almost_equal, assert_, assert_allclose,
                            assert_raises, dec, TestCase)
 from statsmodels.tools.testing import assert_equal
 import statsmodels.sandbox.tsa.fftarma as fa
@@ -2042,6 +2042,141 @@ def test_arima_diff2():
     assert_almost_equal(predicted, predicted_res, 3)
 
 
+def test_arima111_predict_exog_2127():
+    # regression test for issue #2127
+    ef =  [ 0.03005,  0.03917,  0.02828,  0.03644,  0.03379,  0.02744,
+            0.03343,  0.02621,  0.0305 ,  0.02455,  0.03261,  0.03507,
+            0.02734,  0.05373,  0.02677,  0.03443,  0.03331,  0.02741,
+            0.03709,  0.02113,  0.03343,  0.02011,  0.03675,  0.03077,
+            0.02201,  0.04844,  0.05518,  0.03765,  0.05433,  0.03049,
+            0.04829,  0.02936,  0.04421,  0.02457,  0.04007,  0.03009,
+            0.04504,  0.05041,  0.03651,  0.02719,  0.04383,  0.02887,
+            0.0344 ,  0.03348,  0.02364,  0.03496,  0.02549,  0.03284,
+            0.03523,  0.02579,  0.0308 ,  0.01784,  0.03237,  0.02078,
+            0.03508,  0.03062,  0.02006,  0.02341,  0.02223,  0.03145,
+            0.03081,  0.0252 ,  0.02683,  0.0172 ,  0.02225,  0.01579,
+            0.02237,  0.02295,  0.0183 ,  0.02356,  0.02051,  0.02932,
+            0.03025,  0.0239 ,  0.02635,  0.01863,  0.02994,  0.01762,
+            0.02837,  0.02421,  0.01951,  0.02149,  0.02079,  0.02528,
+            0.02575,  0.01634,  0.02563,  0.01719,  0.02915,  0.01724,
+            0.02804,  0.0275 ,  0.02099,  0.02522,  0.02422,  0.03254,
+            0.02095,  0.03241,  0.01867,  0.03998,  0.02212,  0.03034,
+            0.03419,  0.01866,  0.02623,  0.02052]
+    ue =  [  4.9,   5. ,   5. ,   5. ,   4.9,   4.7,   4.8,   4.7,   4.7,
+             4.6,   4.6,   4.7,   4.7,   4.5,   4.4,   4.5,   4.4,   4.6,
+             4.5,   4.4,   4.5,   4.4,   4.6,   4.7,   4.6,   4.7,   4.7,
+             4.7,   5. ,   5. ,   4.9,   5.1,   5. ,   5.4,   5.6,   5.8,
+             6.1,   6.1,   6.5,   6.8,   7.3,   7.8,   8.3,   8.7,   9. ,
+             9.4,   9.5,   9.5,   9.6,   9.8,  10. ,   9.9,   9.9,   9.7,
+             9.8,   9.9,   9.9,   9.6,   9.4,   9.5,   9.5,   9.5,   9.5,
+             9.8,   9.4,   9.1,   9. ,   9. ,   9.1,   9. ,   9.1,   9. ,
+             9. ,   9. ,   8.8,   8.6,   8.5,   8.2,   8.3,   8.2,   8.2,
+             8.2,   8.2,   8.2,   8.1,   7.8,   7.8,   7.8,   7.9,   7.9,
+             7.7,   7.5,   7.5,   7.5,   7.5,   7.3,   7.2,   7.2,   7.2,
+             7. ,   6.7,   6.6,   6.7,   6.7,   6.3,   6.3]
+
+    # rescaling results in convergence failure
+    #model = sm.tsa.ARIMA(np.array(ef)*100, (1,1,1), exog=ue)
+    model = ARIMA(ef, (1,1,1), exog=ue)
+    res = model.fit(transparams=False, iprint=0, disp=0)
+
+    predicts = res.predict(start=len(ef), end = len(ef)+10,
+                           exog=ue[-11:], typ = 'levels')
+
+    # regression test, not verified numbers
+    # if exog=ue in predict, which values are used ?
+    predicts_res = np.array(
+          [ 0.02612291,  0.02361929,  0.024966  ,  0.02448193,  0.0248772 ,
+            0.0248762 ,  0.02506319,  0.02516542,  0.02531214,  0.02544654,
+            0.02559099,  0.02550931])
+
+    # if exog=ue[-11:] in predict
+    predicts_res = np.array(
+          [ 0.02591112,  0.02321336,  0.02436593,  0.02368773,  0.02389767,
+            0.02372018,  0.02374833,  0.02367407,  0.0236443 ,  0.02362868,
+            0.02362312])
+
+    assert_allclose(predicts, predicts_res, atol=1e-6)
+
+
+def test_ARIMA_exog_predict():
+    # test forecasting and dynamic prediction with exog against Stata
+
+    dta = load_macrodata_pandas().data
+    dates = dates_from_range("1959Q1", length=len(dta))
+    cpi_dates = dates_from_range('1959Q1', '2009Q3')
+    dta.index = cpi_dates
+
+    data = dta
+    data['loginv'] = np.log(data['realinv'])
+    data['loggdp'] = np.log(data['realgdp'])
+    data['logcons'] = np.log(data['realcons'])
+
+    forecast_period = dates_from_range('2008Q2', '2009Q3')
+    end = forecast_period[0]
+    data_sample = data.ix[dta.index < end]
+
+    exog_full = data[['loggdp', 'logcons']]
+
+    # pandas
+
+    mod = ARIMA(data_sample['loginv'], (1,0,1), exog=data_sample[['loggdp', 'logcons']])
+    res = mod.fit(disp=0, solver='bfgs', maxiter=5000)
+
+    predicted_arma_fp = res.predict(start=197, end=202, exog=exog_full.values[197:]).values
+    predicted_arma_dp = res.predict(start=193, end=202, exog=exog_full[197:], dynamic=True)
+
+
+    # numpy
+    mod2 = ARIMA(np.asarray(data_sample['loginv']), (1,0,1),
+                   exog=np.asarray(data_sample[['loggdp', 'logcons']]))
+    res2 = mod2.fit(start_params=res.params, disp=0, solver='bfgs', maxiter=5000)
+
+    exog_full = data[['loggdp', 'logcons']]
+    predicted_arma_f = res2.predict(start=197, end=202, exog=exog_full.values[197:])
+    predicted_arma_d = res2.predict(start=193, end=202, exog=exog_full[197:], dynamic=True)
+
+    #ARIMA(1, 1, 1)
+    ex = np.asarray(data_sample[['loggdp', 'logcons']].diff())
+    # The first obsevation is not (supposed to be) used, but I get a Lapack problem
+    # Intel MKL ERROR: Parameter 5 was incorrect on entry to DLASCL.
+    ex[0] = 0
+    mod111 = ARIMA(np.asarray(data_sample['loginv']), (1,1,1),
+                       # Stata differences also the exog
+                       exog=ex)
+
+    res111 = mod111.fit(disp=0, solver='bfgs', maxiter=5000)
+    exog_full_d = data[['loggdp', 'logcons']].diff()
+    res111.predict(start=197, end=202, exog=exog_full_d.values[197:])
+
+    predicted_arima_f = res111.predict(start=196, end=202, exog=exog_full_d.values[197:], typ='levels')
+    predicted_arima_d = res111.predict(start=193, end=202, exog=exog_full_d.values[197:], typ='levels', dynamic=True)
+
+    res_f101 = np.array([ 7.73975859954,  7.71660108543,  7.69808978329,  7.70872117504,
+             7.6518392758 ,  7.69784279784,  7.70290907856,  7.69237782644,
+             7.65017785174,  7.66061689028,  7.65980022857,  7.61505314129,
+             7.51697158428,  7.5165760663 ,  7.5271053284 ])
+    res_f111 = np.array([ 7.74460013693,  7.71958207517,  7.69629561172,  7.71208186737,
+             7.65758850178,  7.69223472572,  7.70411775588,  7.68896109499,
+             7.64016249001,  7.64871881901,  7.62550283402,  7.55814609462,
+             7.44431310053,  7.42963968062,  7.43554675427])
+    res_d111 = np.array([ 7.74460013693,  7.71958207517,  7.69629561172,  7.71208186737,
+             7.65758850178,  7.69223472572,  7.71870821151,  7.7299430215 ,
+             7.71439447355,  7.72544001101,  7.70521902623,  7.64020040524,
+             7.5281927191 ,  7.5149442694 ,  7.52196378005])
+    res_d101 = np.array([ 7.73975859954,  7.71660108543,  7.69808978329,  7.70872117504,
+             7.6518392758 ,  7.69784279784,  7.72522142662,  7.73962377858,
+             7.73245950636,  7.74935432862,  7.74449584691,  7.69589103679,
+             7.5941274688 ,  7.59021764836,  7.59739267775])
+
+    assert_allclose(predicted_arma_dp, res_d101[-len(predicted_arma_d):], atol=1e-4)
+    assert_allclose(predicted_arma_fp, res_f101[-len(predicted_arma_f):], atol=1e-4)
+    assert_allclose(predicted_arma_d, res_d101[-len(predicted_arma_d):], atol=1e-4)
+    assert_allclose(predicted_arma_f, res_f101[-len(predicted_arma_f):], atol=1e-4)
+    assert_allclose(predicted_arima_d, res_d111[-len(predicted_arima_d):], rtol=1e-4, atol=1e-4)
+    assert_allclose(predicted_arima_f, res_f111[-len(predicted_arima_f):], rtol=1e-4, atol=1e-4)
+
+
 if __name__ == "__main__":
     import nose
     nose.runmodule(argv=[__file__, '-vvs', '-x', '--pdb'], exit=False)