Repository: statsmodels/statsmodels

Issue:
BUG: behavior of the extend and append method [SARIMA]
I am surprised by the behavior of the SarimaResult object. Shouldn't running the extend method and the append(refit=false) method with the same data point produce the same forecasts (on using forecast after extend/append)?

I have coded a minimal example here, maybe someone could help me to understand this behaviour. 

[Jupyter Notebook minimal Example](https://github.com/Beich2010/statsmodels_sarima_issue/blob/main/sariam_problem.ipynb)

Thank you in advance for the help and answers.


Discussion:
Thanks for reporting this!  I'll take a look into what's going on.
Is there maybe an workaround without having the timeoverhead from append. 

I need to obtain a constant 24h forecast without using append method, beacause of a big dataset (therefore i needed the extend method). I only need an Arima Model, fitted if possible with Nelder–Mead method. Thank you in advance.

Resolving patch:
diff --git a/statsmodels/tsa/statespace/mlemodel.py b/statsmodels/tsa/statespace/mlemodel.py
--- a/statsmodels/tsa/statespace/mlemodel.py
+++ b/statsmodels/tsa/statespace/mlemodel.py
@@ -1824,9 +1824,14 @@ def _get_extension_time_varying_matrices(
         for name in self.ssm.shapes.keys():
             if name == 'obs' or name in kwargs:
                 continue
-            if getattr(self.ssm, name).shape[-1] > 1:
-                mat = getattr(mod_extend.ssm, name)
-                kwargs[name] = mat[..., -out_of_sample:]
+            original = getattr(self.ssm, name)
+            extended = getattr(mod_extend.ssm, name)
+            so = original.shape[-1]
+            se = extended.shape[-1]
+            if ((so > 1 or se > 1) or (
+                    so == 1 and self.nobs == 1 and
+                    np.any(original[..., 0] != extended[..., 0]))):
+                kwargs[name] = extended[..., -out_of_sample:]
 
         return kwargs
 
diff --git a/statsmodels/tsa/statespace/sarimax.py b/statsmodels/tsa/statespace/sarimax.py
--- a/statsmodels/tsa/statespace/sarimax.py
+++ b/statsmodels/tsa/statespace/sarimax.py
@@ -1741,14 +1741,19 @@ def _get_extension_time_varying_matrices(
         mod_extend.update(params, transformed=transformed,
                           includes_fixed=includes_fixed,)
 
-        # Retrieve the extensions to the time-varying system matrices
-        # and put them in kwargs
+        # Retrieve the extensions to the time-varying system matrices and
+        # put them in kwargs
         for name in self.ssm.shapes.keys():
-            if name == 'obs':
+            if name == 'obs' or name in kwargs:
                 continue
-            if getattr(self.ssm, name).shape[-1] > 1:
-                mat = getattr(mod_extend.ssm, name)
-                kwargs[name] = mat[..., -out_of_sample:]
+            original = getattr(self.ssm, name)
+            extended = getattr(mod_extend.ssm, name)
+            so = original.shape[-1]
+            se = extended.shape[-1]
+            if ((so > 1 or se > 1) or (
+                    so == 1 and self.nobs == 1 and
+                    np.any(original[..., 0] != extended[..., 0]))):
+                kwargs[name] = extended[..., -out_of_sample:]
 
         return kwargs
 


Test patch:
diff --git a/statsmodels/tsa/statespace/tests/test_sarimax.py b/statsmodels/tsa/statespace/tests/test_sarimax.py
--- a/statsmodels/tsa/statespace/tests/test_sarimax.py
+++ b/statsmodels/tsa/statespace/tests/test_sarimax.py
@@ -2591,6 +2591,48 @@ def test_extend_results():
         res2.extend(endog[50:])
 
 
+def test_extend_by_one():
+    endog = np.arange(100)
+    exog = np.ones_like(endog)
+    params = [1., 1., 0.1, 1.]
+
+    mod1 = sarimax.SARIMAX(endog, exog=exog, order=(1, 0, 0), trend='t')
+    res1 = mod1.smooth(params)
+
+    mod2 = sarimax.SARIMAX(endog[:-1], exog=exog[:-1], order=(1, 0, 0),
+                           trend='t')
+    res2 = mod2.smooth(params)
+
+    res3 = res2.extend(endog[-1:], exog=exog[-1:])
+
+    assert_allclose(res3.llf_obs, res1.llf_obs[-1:])
+
+    for attr in [
+            'filtered_state', 'filtered_state_cov', 'predicted_state',
+            'predicted_state_cov', 'forecasts', 'forecasts_error',
+            'forecasts_error_cov', 'standardized_forecasts_error',
+            'forecasts_error_diffuse_cov', 'predicted_diffuse_state_cov',
+            'scaled_smoothed_estimator',
+            'scaled_smoothed_estimator_cov', 'smoothing_error',
+            'smoothed_state',
+            'smoothed_state_cov', 'smoothed_state_autocov',
+            'smoothed_measurement_disturbance',
+            'smoothed_state_disturbance',
+            'smoothed_measurement_disturbance_cov',
+            'smoothed_state_disturbance_cov']:
+        desired = getattr(res1, attr)
+        if desired is not None:
+            desired = desired[..., 99:]
+        assert_equal(getattr(res3, attr), desired)
+
+    assert_allclose(res3.forecast(10, exog=np.ones(10) * 2),
+                    res1.forecast(10, exog=np.ones(10) * 2))
+
+    # Check that we get an error if we try to extend without exog
+    with pytest.raises(ValueError, match='Cloning a model with an exogenous'):
+        res2.extend(endog[-1:])
+
+
 def test_apply_results():
     endog = np.arange(100)
     exog = np.ones_like(endog)