Repository: statsmodels/statsmodels

Issue:
BUG : fail to use time-varying obs_cov
#### Describe the bug
I am trying to use statsmodel to build a Mixed-Frequency VAR model (Mariano & Murasawa. (2010)) , however, but I fail to use time-varying `obs_cov`.  If I replace it with a time-invariant obs_cov, such as `self['obs_cov'] = np.eye(nv)`, it works.


#### Code Sample



    # Mixed-Frequency VAR
    from abc import ABC
    import statsmodels.api as sm
    import numpy as np
    import matplotlib.pyplot as plt
    np.random.seed(42)
    
    
    # Global parameters
    nburn = 0
    nsim = 100
    
    
    class MFVAR(sm.tsa.statespace.MLEModel, ABC):
        def __init__(self, endog):
            dummy_quarterly_obs = dummy_generator()
            n1 = 1   # number of quarterly variables (y)
            n2 = 1   # number of monthly variables (x)
            ns = (n1+n2)*5  # number of state variables
            tlen, nv = endog.shape   # number of observed v. & time length
            # actually, nv = n1+n2
            self.n1, self.n2, self.ns, self.nv = n1, n2, ns, nv
    
            super().__init__(endog, k_states=ns, k_posdef=nv,
                             initialization='stationary')
    
            # 1. time-varying design matrix Z[t]
            z0 = block_diag(1.0/3.0, 1.0, n1, n2)
            z1 = block_diag(2.0/3.0, 0.0, n1, n2)
            z2 = block_diag(1.0, 0.0, n1, n2)
            z3 = block_diag(2.0/3.0, 0.0, n1, n2)
            z4 = block_diag(1.0/3.0, 0.0, n1, n2)
            z = np.column_stack((z0, z1, z2, z3, z4))
            z0_u = block_diag(0.0, 1.0, n1, n2)
            z_u = np.column_stack((z0_u, np.zeros((nv, nv*4))))
            z_time = np.zeros((nv, ns, tlen))
            for t in range(tlen):
                if dummy_quarterly_obs[t]:
                    z_time[:, :, t] = z
                else:
                    z_time[:, :, t] = z_u
            self['design'] = z_time
    
            # # 2. time-varying observation disturbance H[t]
            # when y is observable
            h = np.zeros((nv, nv))
            # when y is unobservable
            h_u = block_diag(1.0, 0.0, n1, n2)  # random variable N(0, 1)
            h_time = np.zeros((nv, nv, tlen))
            for t in range(tlen):
                if dummy_quarterly_obs[t]:
                    h_time[:, :, t] = h
                else:
                    h_time[:, :, t] = h_u
            self['obs_cov'] = h_time
            # self['obs_cov'] = np.eye(nv)
    
            # 3. time-invariant selection matrix R
            self['selection'] = np.eye(ns, nv)
    
        def update(self, params, transformed=True, **kwargs):
            n1, n2, ns, nv = self.n1, self.n2, self.ns, self.nv
            # nv = n1 + n2
            # ns = 5 * nv
    
            # 4. time-invariant transition matrix T
            var_coef = params[:nv**2].reshape(nv, nv)
            var_coef = np.column_stack((var_coef, np.zeros((nv, nv*4))))
            self['transition'] = np.row_stack((var_coef, np.eye(nv*4, ns)))
    
            # 5. time-invariant state covariance Q
            tril_state_cov = vec_tril(params[nv**2:])   # lower-triangular matrix
            self['state_cov'] = tril_state_cov @ tril_state_cov.transpose()
    
        @property
        def start_params(self):
            phi_11, phi_12, phi_21, phi_22 = 0.4, 0., 0., 0.8   # initial guess
            sig_11, sig_12, sig_22 = 1.0, 0., 1.0
            return np.array([phi_11, phi_12, phi_21, phi_22, sig_11, sig_12, sig_22])
    
    
    def block_diag(a, b, n1, n2):
        """
        Create block diagonal matrix of this form:
         a*I |  0
        -----+-----
         0   |  b*I
         The size of the upper left corner is (n1, n1)
         The size of the lower right corner is (n2, n2)
        """
        upper_left = np.identity(n1)*a
        lower_right = np.identity(n2)*b
        upper_right = np.zeros((n1, n2))
        lower_left = np.zeros((n2, n1))
        upper = np.column_stack((upper_left, upper_right))
        lower = np.column_stack((lower_left, lower_right))
        z0 = np.row_stack((upper, lower))
        return z0
    
    
    def vec_tril(x, order='C'):
        """
        Reverse half-vectorization into a triangular matrix in the order of C or F(ortran)
        If x is complex, ivech(x) is conjugate symmetric.
        That is, ivech(x) = ivech(x).transpose().conj()
        """
        r = len(x)
        n = np.int((np.sqrt(1+8*r)-1)/2)
        if all(np.isreal(x)):
            a_mat = np.zeros((n, n))
        else:
            a_mat = np.zeros((n, n), 'complex')
        if order == 'C':
            d = np.tril_indices(n)
            a_mat[d] = x
        else:
            a_mat[:, 0] = x[0:n]
            oldindex = n
            for ii in np.arange(1, n):
                newindex = oldindex + n - ii
                a_mat[ii:n, ii] = x[oldindex:newindex]
                oldindex = newindex
        return a_mat
    
    
    def dummy_generator():
        dummy = [1, 0, 0]*nsim
        dummy = np.array(dummy, dtype=int)
        return dummy
    
    
    def data_generator():
        # % artificial data
        phi_11, phi_12, phi_21, phi_22 = 0.5, 0., 0., 0.5  # initial guess
        sig_11, sig_12, sig_22 = 0.5, 0.0, 0.5
        params = [phi_11, phi_12, phi_21, phi_22, sig_11, sig_12, sig_22]
        params = np.array(params)
        model_true = MFVAR(np.zeros((nsim*3, 2)))
        consumption = model_true.simulate(params, nsim*3)  # % consumption
        # consumption = np.delete(consumption, np.arange(burn))
        return consumption
    
    
    # SS fit
    data_consumption = data_generator()
    model = MFVAR(data_consumption)
    res = model.fit()
    print(res.summary())
    

    



### Your code here that produces the bug
 File "C:\Users\Anaconda3\lib\site-packages\statsmodels\tsa\statespace\simulation_smoother.py", line 169, in _simulate
    np.r_[measurement_shocks.ravel(), state_shocks.ravel()],

AttributeError: 'NoneType' object has no attribute 'ravel'



Discussion:
This looks like it was maybe fixed in #6280. Can you try updating your statsmodels to v0.11 and see if that fixes the problem?
Well I guess in v0.11 it raises a different exception. I'll take a look
Indeed this is a bug, still in v0.11.  Thanks for the report!

Resolving patch:
diff --git a/statsmodels/tsa/statespace/initialization.py b/statsmodels/tsa/statespace/initialization.py
--- a/statsmodels/tsa/statespace/initialization.py
+++ b/statsmodels/tsa/statespace/initialization.py
@@ -539,7 +539,7 @@ def __call__(self, index=None, model=None, initial_state_mean=None,
             state_intercept = model['state_intercept', ix1, 0]
             transition = model[('transition',) + ix2 + (0,)]
             selection = model['selection', ix1, :, 0]
-            state_cov = model['state_cov']
+            state_cov = model['state_cov', :, :, 0]
             selected_state_cov = np.dot(selection, state_cov).dot(selection.T)
 
         # Create output arrays if not given
diff --git a/statsmodels/tsa/statespace/kalman_filter.py b/statsmodels/tsa/statespace/kalman_filter.py
--- a/statsmodels/tsa/statespace/kalman_filter.py
+++ b/statsmodels/tsa/statespace/kalman_filter.py
@@ -1157,12 +1157,6 @@ def simulate(self, nsimulations, measurement_shocks=None,
             measurement_shocks = np.random.multivariate_normal(
                 mean=np.zeros(self.k_endog), cov=self['obs_cov'],
                 size=nsimulations)
-        elif state_shocks is not None:
-            measurement_shocks = np.zeros((nsimulations, self.k_endog))
-            for i in range(nsimulations):
-                measurement_shocks[i] = np.random.multivariate_normal(
-                    mean=np.zeros(self.k_endog),
-                    cov=self['obs_cov', ..., i], size=nsimulations)
 
         # Check / generate state shocks
         if state_shocks is not None:
@@ -1183,12 +1177,22 @@ def simulate(self, nsimulations, measurement_shocks=None,
             state_shocks = np.random.multivariate_normal(
                 mean=np.zeros(self.k_posdef), cov=self['state_cov'],
                 size=nsimulations)
-        elif measurement_shocks is not None:
+
+        # Handle time-varying case
+        tvp = (self.shapes['obs_cov'][-1] > 1 or
+               self.shapes['state_cov'][-1] > 1)
+        if tvp and measurement_shocks is None:
+            measurement_shocks = np.zeros((nsimulations, self.k_endog))
+            for i in range(nsimulations):
+                measurement_shocks[i] = np.random.multivariate_normal(
+                    mean=np.zeros(self.k_endog),
+                    cov=self['obs_cov', ..., i])
+        if tvp and state_shocks is None:
             state_shocks = np.zeros((nsimulations, self.k_posdef))
             for i in range(nsimulations):
                 state_shocks[i] = np.random.multivariate_normal(
                     mean=np.zeros(self.k_posdef),
-                    cov=self['state_cov', ..., i], size=nsimulations)
+                    cov=self['state_cov', ..., i])
 
         # Get the initial states
         if initial_state is not None:


Test patch:
diff --git a/statsmodels/tsa/statespace/tests/test_simulate.py b/statsmodels/tsa/statespace/tests/test_simulate.py
--- a/statsmodels/tsa/statespace/tests/test_simulate.py
+++ b/statsmodels/tsa/statespace/tests/test_simulate.py
@@ -7,10 +7,11 @@
 
 import numpy as np
 import pandas as pd
-from numpy.testing import assert_, assert_allclose
+from numpy.testing import assert_, assert_allclose, assert_equal
 import pytest
 from scipy.signal import lfilter
 
+from .test_impulse_responses import TVSS
 from statsmodels.tools.sm_exceptions import SpecificationWarning, \
     EstimationWarning
 from statsmodels.tsa.statespace import (sarimax, structural, varmax,
@@ -1740,3 +1741,46 @@ def test_pandas_anchor():
                           initial_state=np.zeros(1))
     assert_allclose(actual, desired)
     assert_(actual.index.equals(desired.index))
+
+
+@pytest.mark.smoke
+def test_time_varying(reset_randomstate):
+    mod = TVSS(np.zeros((10, 2)))
+    mod.simulate([], 10)
+
+
+def test_time_varying_obs_cov(reset_randomstate):
+    mod = TVSS(np.zeros((10, 2)))
+    mod['obs_cov'] = np.zeros((mod.k_endog, mod.k_endog, mod.nobs))
+    mod['obs_cov', ..., 9] = np.eye(mod.k_endog)
+    mod['state_intercept', :] = 0
+    mod['state_cov'] = mod['state_cov', :, :, 0] * 0
+    mod['selection'] = mod['selection', :, :, 0]
+    assert_equal(mod['state_cov'].shape, (mod.ssm.k_posdef, mod.ssm.k_posdef))
+    assert_equal(mod['selection'].shape, (mod.k_states, mod.ssm.k_posdef))
+
+    sim = mod.simulate([], 10, initial_state=np.zeros(mod.k_states))
+    assert_allclose(sim[:9], mod['obs_intercept', :, :9].T)
+
+
+def test_time_varying_state_cov(reset_randomstate):
+    mod = TVSS(np.zeros((10, 2)))
+    mod['obs_cov'] = mod['obs_cov', :, :, 0] * 0
+    mod['selection'] = mod['selection', :, :, 0]
+    mod['state_intercept', :] = 0
+    mod['state_cov'] = np.zeros((mod.ssm.k_posdef, mod.ssm.k_posdef, mod.nobs))
+    mod['state_cov', ..., -1] = np.eye(mod.ssm.k_posdef)
+    assert_equal(mod['obs_cov'].shape, (mod.k_endog, mod.k_endog))
+    assert_equal(mod['selection'].shape, (mod.k_states, mod.ssm.k_posdef))
+    sim = mod.simulate([], 10)
+    assert_allclose(sim, mod['obs_intercept'].T)
+
+
+@pytest.mark.smoke
+def test_time_varying_selection(reset_randomstate):
+    mod = TVSS(np.zeros((10, 2)))
+    mod['obs_cov'] = mod['obs_cov', :, :, 0]
+    mod['state_cov'] = mod['state_cov', :, :, 0]
+    assert_equal(mod['obs_cov'].shape, (mod.k_endog, mod.k_endog))
+    assert_equal(mod['state_cov'].shape, (mod.ssm.k_posdef, mod.ssm.k_posdef))
+    mod.simulate([], 10)