Repository: statsmodels/statsmodels

Issue:
ARIMA forecasts are hard-coded for d=1
Just need to fix the unintegrate to take into account two-levels I'm pretty sure. Ditto forecast errors and confidence intervals. 



Discussion:

Resolving patch:
diff --git a/statsmodels/tsa/arima_model.py b/statsmodels/tsa/arima_model.py
--- a/statsmodels/tsa/arima_model.py
+++ b/statsmodels/tsa/arima_model.py
@@ -25,7 +25,7 @@
 from statsmodels.tsa.tsatools import (lagmat, add_trend,
                                       _ar_transparams, _ar_invtransparams,
                                       _ma_transparams, _ma_invtransparams,
-                                      unintegrate)
+                                      unintegrate, unintegrate_levels)
 from statsmodels.tsa.vector_ar import util
 from statsmodels.tsa.ar_model import AR
 from statsmodels.tsa.arima_process import arma2ma
@@ -240,6 +240,15 @@
                       }
 
 
+def cumsum_n(x, n):
+    if n:
+        n -= 1
+        x = np.cumsum(x)
+        return cumsum_n(x, n)
+    else:
+        return x
+
+
 def _check_arima_start(start, k_ar, k_diff, method, dynamic):
     if start < 0:
         raise ValueError("The start index %d of the original series "
@@ -984,9 +993,13 @@ def __new__(cls, endog, order, exog=None, dates=None, freq=None,
     def __init__(self, endog, order, exog=None, dates=None, freq=None,
                  missing='none'):
         p, d, q = order
+        if d > 2:
+            #NOTE: to make more general, need to address the d == 2 stuff
+            # in the predict method
+            raise ValueError("d > 2 is not supported")
         super(ARIMA, self).__init__(endog, (p, q), exog, dates, freq, missing)
         self.k_diff = d
-        self._first_levels = self.endog[:d]
+        self._first_unintegrate = unintegrate_levels(self.endog[:d], d)
         self.endog = np.diff(self.endog, n=d)
         #NOTE: will check in ARMA but check again since differenced now
         _check_estimable(len(self.endog), p+q)
@@ -1148,23 +1161,38 @@ def predict(self, params, start=None, end=None, exog=None, typ='linear',
 
                 start = self._get_predict_start(start, dynamic)
                 end, out_of_sample = self._get_predict_end(end)
+                d = self.k_diff
                 if 'mle' in self.method:
+                    start += d - 1  # for case where d == 2
+                    end += d - 1
                     # add each predicted diff to lagged endog
                     if out_of_sample:
                         fv = predict[:-out_of_sample] + endog[start:end+1]
-                        fv = np.r_[fv, endog[-1] +
-                                   np.cumsum(predict[-out_of_sample:])]
+                        if d == 2:  #TODO: make a general solution to this
+                            fv += np.diff(endog[start - 1:end + 1])
+                        levels = unintegrate_levels(endog[-d:], d)
+                        fv = np.r_[fv,
+                                   unintegrate(predict[-out_of_sample:],
+                                               levels)[d:]]
                     else:
                         fv = predict + endog[start:end + 1]
+                        if d == 2:
+                            fv += np.diff(endog[start - 1:end + 1])
                 else:
                     k_ar = self.k_ar
                     if out_of_sample:
                         fv = (predict[:-out_of_sample] +
                               endog[max(start, self.k_ar-1):end+k_ar+1])
-                        fv = np.r_[fv, endog[-1] +
-                                   np.cumsum(predict[-out_of_sample:])]
+                        if d == 2:
+                            fv += np.diff(endog[start - 1:end + 1])
+                        levels = unintegrate_levels(endog[-d:], d)
+                        fv = np.r_[fv,
+                                   unintegrate(predict[-out_of_sample:],
+                                               levels)[d:]]
                     else:
                         fv = predict + endog[max(start, k_ar):end+k_ar+1]
+                        if d == 2:
+                            fv += np.diff(endog[start - 1:end + 1])
             else:
                 #IFF we need to use pre-sample values assume pre-sample
                 # residuals are zero, do this by a hack
@@ -1759,7 +1787,8 @@ def _forecast_error(self, steps):
         sigma2 = self.sigma2
         ma_rep = arma2ma(np.r_[1, -self.arparams],
                          np.r_[1, self.maparams], nobs=steps)
-        fcerr = np.sqrt(np.cumsum(np.cumsum(ma_rep)**2)*sigma2)
+
+        fcerr = np.sqrt(np.cumsum(cumsum_n(ma_rep, self.k_diff)**2)*sigma2)
         return fcerr
 
     def _forecast_conf_int(self, forecast, fcerr, alpha):
@@ -1810,7 +1839,11 @@ def forecast(self, steps=1, exog=None, alpha=.05):
                                                self.k_trend, self.k_exog,
                                                self.model.endog,
                                                exog, method=self.model.method)
-        forecast = self.model.data.endog[-1] + np.cumsum(forecast)
+
+        d = self.k_diff
+        endog = self.model.data.endog[-d:]
+        forecast = unintegrate(forecast, unintegrate_levels(endog, d))[d:]
+
         # get forecast errors
         fcerr = self._forecast_error(steps)
         conf_int = self._forecast_conf_int(forecast, fcerr, alpha)
@@ -1851,7 +1884,8 @@ def plot_predict(self, start=None, end=None, exog=None, dynamic=False,
             import re
             k_diff = self.k_diff
             label = re.sub("D\d*\.", "", self.model.endog_names)
-            levels = unintegrate(self.model.endog, self.model._first_levels)
+            levels = unintegrate(self.model.endog,
+                                 self.model._first_unintegrate)
             ax.plot(x[:end + 1 - start],
                     levels[start + k_diff:end + k_diff + 1], label=label)
 
diff --git a/statsmodels/tsa/tsatools.py b/statsmodels/tsa/tsatools.py
--- a/statsmodels/tsa/tsatools.py
+++ b/statsmodels/tsa/tsatools.py
@@ -547,6 +547,32 @@ def _ma_invtransparams(macoefs):
     invmacoefs = -np.log((1-macoefs)/(1+macoefs))
     return invmacoefs
 
+
+def unintegrate_levels(x, d):
+    """
+    Returns the successive differences needed to unintegrate the series.
+
+    Parameters
+    ----------
+    x : array-like
+        The original series
+    d : int
+        The number of differences of the differenced series.
+
+    Returns
+    -------
+    y : array-like
+        The increasing differences from 0 to d-1 of the first d elements
+        of x.
+
+    See Also
+    --------
+    unintegrate
+    """
+    x = x[:d]
+    return np.asarray([np.diff(x, d - i)[0] for i in range(d, 0, -1)])
+
+
 def unintegrate(x, levels):
     """
     After taking n-differences of a series, return the original series
@@ -567,14 +593,16 @@ def unintegrate(x, levels):
     Examples
     --------
     >>> x = np.array([1, 3, 9., 19, 8.])
-    >>> levels = [x[0], np.diff(x, 1)[0]]
-    >>> _unintegrate(np.diff(x, 2), levels)
+    >>> levels = unintegrate_levels(x, 2)
+    >>> levels
+    array([ 1.,  2.])
+    >>> unintegrate(np.diff(x, 2), levels)
     array([  1.,   3.,   9.,  19.,   8.])
     """
-    levels = levels[:] # copy
+    levels = list(levels)[:] # copy
     if len(levels) > 1:
         x0 = levels.pop(-1)
-        return _unintegrate(np.cumsum(np.r_[x0, x]), levels)
+        return unintegrate(np.cumsum(np.r_[x0, x]), levels)
     x0 = levels[0]
     return np.cumsum(np.r_[x0, x])
 


Test patch:
diff --git a/statsmodels/tsa/tests/test_arima.py b/statsmodels/tsa/tests/test_arima.py
--- a/statsmodels/tsa/tests/test_arima.py
+++ b/statsmodels/tsa/tests/test_arima.py
@@ -12,6 +12,8 @@
 from .results import results_arma, results_arima
 import os
 from statsmodels.tsa.arima_process import arma_generate_sample
+from statsmodels.datasets.macrodata import load as load_macrodata
+from statsmodels.datasets.macrodata import load_pandas as load_macrodata_pandas
 import pandas
 
 try:
@@ -551,8 +553,7 @@ class Test_ARIMA111(CheckArimaResultsMixin, CheckForecastMixin,
                     CheckDynamicForecastMixin):
     @classmethod
     def setupClass(cls):
-        from statsmodels.datasets.macrodata import load
-        cpi = load().data['cpi']
+        cpi = load_macrodata().data['cpi']
         cls.res1 = ARIMA(cpi, (1,1,1)).fit(disp=-1)
         cls.res2 = results_arima.ARIMA111()
         # make sure endog names changes to D.cpi
@@ -583,8 +584,7 @@ class Test_ARIMA111CSS(CheckArimaResultsMixin, CheckForecastMixin,
                        CheckDynamicForecastMixin):
     @classmethod
     def setupClass(cls):
-        from statsmodels.datasets.macrodata import load
-        cpi = load().data['cpi']
+        cpi = load_macrodata().data['cpi']
         cls.res1 = ARIMA(cpi, (1,1,1)).fit(disp=-1, method='css')
         cls.res2 = results_arima.ARIMA111(method='css')
         cls.res2.fittedvalues = - cpi[1:-1] + cls.res2.linear
@@ -615,8 +615,7 @@ def setupClass(cls):
 class Test_ARIMA112CSS(CheckArimaResultsMixin):
     @classmethod
     def setupClass(cls):
-        from statsmodels.datasets.macrodata import load
-        cpi = load().data['cpi']
+        cpi = load_macrodata().data['cpi']
         cls.res1 = ARIMA(cpi, (1,1,2)).fit(disp=-1, method='css',
                                 start_params = [.905322, -.692425, 1.07366,
                                                 0.172024])
@@ -653,10 +652,9 @@ def test_freq(self):
 #class Test_ARIMADates(CheckArmaResults, CheckForecast, CheckDynamicForecast):
 #    @classmethod
 #    def setupClass(cls):
-#        from statsmodels.datasets.macrodata import load
 #        from statsmodels.tsa.datetools import dates_from_range
 #
-#        cpi = load().data['cpi']
+#        cpi = load_macrodata().data['cpi']
 #        dates = dates_from_range('1959q1', length=203)
 #        cls.res1 = ARIMA(cpi, dates=dates, freq='Q').fit(order=(1,1,1), disp=-1)
 #        cls.res2 = results_arima.ARIMA111()
@@ -670,8 +668,7 @@ def test_freq(self):
 
 
 def test_arima_predict_mle_dates():
-    from statsmodels.datasets.macrodata import load
-    cpi = load().data['cpi']
+    cpi = load_macrodata().data['cpi']
     res1 = ARIMA(cpi, (4,1,1), dates=cpi_dates, freq='Q').fit(disp=-1)
 
     arima_forecasts = np.genfromtxt(open(
@@ -723,8 +720,7 @@ def test_arma_predict_mle_dates():
 
 
 def test_arima_predict_css_dates():
-    from statsmodels.datasets.macrodata import load
-    cpi = load().data['cpi']
+    cpi = load_macrodata().data['cpi']
     res1 = ARIMA(cpi, (4,1,1), dates=cpi_dates, freq='Q').fit(disp=-1,
             method='css', trend='nc')
 
@@ -772,8 +768,7 @@ def test_arma_predict_css_dates():
 
 
 def test_arima_predict_mle():
-    from statsmodels.datasets.macrodata import load
-    cpi = load().data['cpi']
+    cpi = load_macrodata().data['cpi']
     res1 = ARIMA(cpi, (4,1,1)).fit(disp=-1)
     # fit the model so that we get correct endog length but use
 
@@ -1001,8 +996,7 @@ def test_arma_predict_indices():
 
 
 def test_arima_predict_indices():
-    from statsmodels.datasets.macrodata import load
-    cpi = load().data['cpi']
+    cpi = load_macrodata().data['cpi']
     model = ARIMA(cpi, (4,1,1), dates=cpi_dates, freq='Q')
     model.method = 'mle'
 
@@ -1133,8 +1127,7 @@ def test_arima_predict_indices():
 
 
 def test_arima_predict_indices_css():
-    from statsmodels.datasets.macrodata import load
-    cpi = load().data['cpi']
+    cpi = load_macrodata().data['cpi']
     #NOTE: Doing no-constant for now to kick the conditional exogenous
     #issue 274 down the road
     # go ahead and git the model to set up necessary variables
@@ -1148,8 +1141,7 @@ def test_arima_predict_indices_css():
 
 
 def test_arima_predict_css():
-    from statsmodels.datasets.macrodata import load
-    cpi = load().data['cpi']
+    cpi = load_macrodata().data['cpi']
     #NOTE: Doing no-constant for now to kick the conditional exogenous
     #issue 274 down the road
     # go ahead and git the model to set up necessary variables
@@ -1288,8 +1280,7 @@ def test_arima_predict_css():
 
 def test_arima_predict_css_diffs():
 
-    from statsmodels.datasets.macrodata import load
-    cpi = load().data['cpi']
+    cpi = load_macrodata().data['cpi']
     #NOTE: Doing no-constant for now to kick the conditional exogenous
     #issue 274 down the road
     # go ahead and git the model to set up necessary variables
@@ -1433,8 +1424,7 @@ def test_arima_predict_css_diffs():
 
 def test_arima_predict_mle_diffs():
 
-    from statsmodels.datasets.macrodata import load
-    cpi = load().data['cpi']
+    cpi = load_macrodata().data['cpi']
     #NOTE: Doing no-constant for now to kick the conditional exogenous
     #issue 274 down the road
     # go ahead and git the model to set up necessary variables
@@ -1573,9 +1563,8 @@ def test_arima_predict_mle_diffs():
 
 
 def test_arima_wrapper():
-    from statsmodels.datasets.macrodata import load_pandas
 
-    cpi = load_pandas().data['cpi']
+    cpi = load_macrodata_pandas().data['cpi']
     cpi.index = pandas.Index(cpi_dates)
     res = ARIMA(cpi, (4,1,1), freq='Q').fit(disp=-1)
     assert_equal(res.params.index, pandas.Index(['const', 'ar.L1.D.cpi', 'ar.L2.D.cpi',
@@ -1586,8 +1575,7 @@ def test_arima_wrapper():
 
 def test_1dexog():
     # smoke test, this will raise an error if broken
-    from statsmodels.datasets.macrodata import load_pandas
-    dta = load_pandas().data
+    dta = load_macrodata_pandas().data
     endog = dta['realcons'].values
     exog = dta['m1'].values.squeeze()
     mod = ARMA(endog, (1,1), exog).fit(disp=-1)
@@ -1604,8 +1592,7 @@ def test_arima_predict_bug():
 
 def test_arima_predict_q2():
     # bug with q > 1 for arima predict
-    from statsmodels.datasets import macrodata
-    inv = macrodata.load().data['realinv']
+    inv = load_macrodata().data['realinv']
     arima_mod = ARIMA(np.log(inv), (1,1,2)).fit(start_params=[0,0,0,0], disp=-1)
     fc, stderr, conf_int = arima_mod.forecast(5)
     # values copy-pasted from gretl
@@ -1744,8 +1731,7 @@ def test_arima_predict_noma():
 
 
 def test_arimax():
-    from statsmodels.datasets.macrodata import load_pandas
-    dta = load_pandas().data
+    dta = load_macrodata_pandas().data
     dates = dates_from_range("1959Q1", length=len(dta))
     dta.index = cpi_dates
     dta = dta[["realdpi", "m1", "realgdp"]]
@@ -1806,8 +1792,7 @@ def test_bad_start_params():
     mod = ARMA(endog, (15, 0))
     assert_raises(ValueError, mod.fit)
 
-    from statsmodels.datasets.macrodata import load
-    inv = load().data['realinv']
+    inv = load_macrodata().data['realinv']
     arima_mod = ARIMA(np.log(inv), (1,1,2))
     assert_raises(ValueError, mod.fit)
 
@@ -2018,6 +2003,44 @@ def test_plot_predict():
     plt.close(fig)
 
 
+
+def test_arima_diff2():
+    dta = load_macrodata_pandas().data['cpi']
+    dates = dates_from_range("1959Q1", length=len(dta))
+    dta.index = cpi_dates
+    mod = ARIMA(dta, (3, 2, 1)).fit(disp=-1)
+    fc, fcerr, conf_int = mod.forecast(10)
+    # forecasts from gretl
+    conf_int_res = [ (216.139,  219.231),
+                     (216.472,  221.520),
+                     (217.064,  223.649),
+                     (217.586,  225.727),
+                     (218.119,  227.770),
+                     (218.703,  229.784),
+                     (219.306,  231.777),
+                     (219.924,  233.759),
+                     (220.559,  235.735),
+                     (221.206,  237.709)]
+
+
+    fc_res = [217.685, 218.996, 220.356, 221.656, 222.945, 224.243, 225.541,
+          226.841, 228.147, 229.457]
+    fcerr_res = [0.7888, 1.2878, 1.6798, 2.0768,  2.4620, 2.8269, 3.1816,
+                 3.52950, 3.8715, 4.2099]
+
+    assert_almost_equal(fc, fc_res, 3)
+    assert_almost_equal(fcerr, fcerr_res, 3)
+    assert_almost_equal(conf_int, conf_int_res, 3)
+
+    predicted = mod.predict('2008Q1', '2012Q1', typ='levels')
+
+    predicted_res = [214.464, 215.478, 221.277, 217.453, 212.419, 213.530,
+                     215.087, 217.685 , 218.996 , 220.356 , 221.656 ,
+                     222.945 , 224.243 , 225.541 , 226.841 , 228.147 ,
+                     229.457]
+    assert_almost_equal(predicted, predicted_res, 3)
+
+
 if __name__ == "__main__":
     import nose
     nose.runmodule(argv=[__file__, '-vvs', '-x', '--pdb'], exit=False)