Repository: statsmodels/statsmodels

Issue:
SARIMAX predict breaks when simple_differencing=True
In the diagnostics branch that is awaiting merge, sarimax.predict breaks when simple_differencing=True. You get the size matching error like

ValueError: Invalid dimensions for time-varying state intercept vector. Requires shape (*,673), got (50, 648)

This seems to be related to not adjusting properly for the points dropped from the prior differencing. Which seems to affect other things too. For example, the new plot_diagnostics has a similar problem when using simple_differencing. Seems that it may be related to loglikelihood_burn not being set properly?



Discussion:

Resolving patch:
diff --git a/statsmodels/tsa/statespace/mlemodel.py b/statsmodels/tsa/statespace/mlemodel.py
--- a/statsmodels/tsa/statespace/mlemodel.py
+++ b/statsmodels/tsa/statespace/mlemodel.py
@@ -297,7 +297,7 @@ def filter(self, params, transformed=True, cov_type=None, cov_kwds=None,
             Additional keyword arguments to pass to the Kalman filter. See
             `KalmanFilter.filter` for more details.
         """
-        params = np.array(params)
+        params = np.array(params, ndmin=1)
 
         if not transformed:
             params = self.transform_params(params)
@@ -702,7 +702,7 @@ def transform_params(self, unconstrained):
         This is a noop in the base class, subclasses should override where
         appropriate.
         """
-        return np.array(unconstrained)
+        return np.array(unconstrained, ndmin=1)
 
     def untransform_params(self, constrained):
         """
@@ -749,7 +749,7 @@ def update(self, params, transformed=True):
         Since Model is a base class, this method should be overridden by
         subclasses to perform actual updating steps.
         """
-        params = np.array(params)
+        params = np.array(params, ndmin=1)
 
         if not transformed:
             params = self.transform_params(params)
diff --git a/statsmodels/tsa/statespace/representation.py b/statsmodels/tsa/statespace/representation.py
--- a/statsmodels/tsa/statespace/representation.py
+++ b/statsmodels/tsa/statespace/representation.py
@@ -358,6 +358,8 @@ def __getitem__(self, key):
             # See note on time-varying arrays, below
             if matrix.shape[-1] == 1:
                 return matrix[[slice(None)]*(matrix.ndim-1) + [0]]
+            else:
+                return matrix
         # Otherwise if we have a tuple, we want a slice of a matrix
         elif _type is tuple:
             name, slice_ = key[0], key[1:]
diff --git a/statsmodels/tsa/statespace/sarimax.py b/statsmodels/tsa/statespace/sarimax.py
--- a/statsmodels/tsa/statespace/sarimax.py
+++ b/statsmodels/tsa/statespace/sarimax.py
@@ -554,6 +554,10 @@ def prepare_data(self):
                 exog = diff(exog.copy(), self.orig_k_diff,
                             self.orig_k_seasonal_diff, self.k_seasons)
 
+            # Reset the ModelData datasets
+            self.data.endog, self.data.exog = (
+                self.data._convert_endog_exog(endog, exog))
+
         # Reset the nobs
         self.nobs = endog.shape[0]
 
@@ -842,7 +846,7 @@ def initial_selection(self):
 
     def filter(self, params, transformed=True, cov_type=None, return_ssm=False,
                **kwargs):
-        params = np.array(params)
+        params = np.array(params, ndmin=1)
 
         # Transform parameters if necessary
         if not transformed:
@@ -1291,7 +1295,7 @@ def transform_params(self, unconstrained):
         polynomials, although it only excludes a very small portion very close
         to the invertibility boundary.
         """
-        unconstrained = np.array(unconstrained)
+        unconstrained = np.array(unconstrained, ndmin=1)
         constrained = np.zeros(unconstrained.shape, unconstrained.dtype)
 
         start = end = 0
@@ -1395,7 +1399,7 @@ def untransform_params(self, constrained):
         polynomials, although it only excludes a very small portion very close
         to the invertibility boundary.
         """
-        constrained = np.array(constrained)
+        constrained = np.array(constrained, ndmin=1)
         unconstrained = np.zeros(constrained.shape, constrained.dtype)
 
         start = end = 0
@@ -1875,7 +1879,7 @@ def predict(self, start=None, end=None, exog=None, dynamic=False,
         # Handle exogenous parameters
         if _out_of_sample and (self.model.k_exog + self.model.k_trend > 0):
             # Create a new faux SARIMAX model for the extended dataset
-            nobs = self.model.orig_endog.shape[0] + _out_of_sample
+            nobs = self.model.data.orig_endog.shape[0] + _out_of_sample
             endog = np.zeros((nobs, self.model.k_endog))
 
             if self.model.k_exog > 0:
@@ -1891,7 +1895,7 @@ def predict(self, start=None, end=None, exog=None, dynamic=False,
                                      ' appropriate shape. Required %s, got %s.'
                                      % (str(required_exog_shape),
                                         str(exog.shape)))
-                exog = np.c_[self.model.orig_exog.T, exog.T].T
+                exog = np.c_[self.model.data.orig_exog.T, exog.T].T
 
             # TODO replace with init_kwds or specification or similar
             model = SARIMAX(


Test patch:
diff --git a/statsmodels/tsa/statespace/tests/test_representation.py b/statsmodels/tsa/statespace/tests/test_representation.py
--- a/statsmodels/tsa/statespace/tests/test_representation.py
+++ b/statsmodels/tsa/statespace/tests/test_representation.py
@@ -459,43 +459,6 @@ def __init__(self):
         self.run_filter()
 
 
-class TestClark1989ConserveAll(Clark1989):
-    """
-    Memory conservation forecasting test for the loglikelihood and filtered
-    states.
-    """
-    def __init__(self):
-        super(TestClark1989ConserveAll, self).__init__(
-            dtype=float, conserve_memory=0x01 | 0x02 | 0x04 | 0x08
-        )
-        # self.model.loglikelihood_burn = self.true['start']
-        self.model.loglikelihood_burn = 0
-        self.run_filter()
-
-    def test_loglike(self):
-        assert_almost_equal(
-            self.results.llf_obs[0], self.true['loglike'], 2
-        )
-
-    def test_filtered_state(self):
-        end = self.true_states.shape[0]
-        assert_almost_equal(
-            self.results.filtered_state[0][-1],
-            self.true_states.iloc[end-1, 0], 4
-        )
-        assert_almost_equal(
-            self.results.filtered_state[1][-1],
-            self.true_states.iloc[end-1, 1], 4
-        )
-        assert_almost_equal(
-            self.results.filtered_state[4][-1],
-            self.true_states.iloc[end-1, 2], 4
-        )
-        assert_almost_equal(
-            self.results.filtered_state[5][-1],
-            self.true_states.iloc[end-1, 3], 4
-        )
-
 class TestClark1989ConserveAll(Clark1989):
     """
     Memory conservation forecasting test for the loglikelihood and filtered
diff --git a/statsmodels/tsa/statespace/tests/test_sarimax.py b/statsmodels/tsa/statespace/tests/test_sarimax.py
--- a/statsmodels/tsa/statespace/tests/test_sarimax.py
+++ b/statsmodels/tsa/statespace/tests/test_sarimax.py
@@ -872,8 +872,15 @@ def test_transform_untransform(self):
     def test_results(self):
         self.result = self.model.filter(self.true_params)
 
-        # Just make sure that no exceptions are thrown during summary
+        # Just make sure that no exceptions are thrown during summary, predict
+        # forecast
         self.result.summary()
+        self.result.predict()
+        if self.model.k_exog == 0:
+            self.result.forecast()
+        else:
+            exog = np.r_[[0]*self.model.k_exog].reshape(self.model.k_exog, 1)
+            self.result.forecast(exog=exog)
 
         # And make sure no expections are thrown calculating any of the
         # covariance matrix types