Repository: statsmodels/statsmodels

Issue:
BUG: Statespace: Test failures on certain Travis builds.
Test failures on recent Travis builds. It seems that differences in the underlying fortran libraries / routines is causing precision issues. In particular, the following:
- `test_representation.py:TestClark1989.test_kalman_gain` is not `assert_allclose` at `atol=1e-5`. Can be fixed with `atol=1e-4`.
- Various SARIMAX coverages tests have nearly singular covariance matrices (this is not unexpected, they are not necessarily good models for the dataset, they are just used for coverage). This can be fixed by using the psuedo inverse for inverting covariance parameter matrices.



Discussion:

Resolving patch:
diff --git a/statsmodels/tsa/statespace/mlemodel.py b/statsmodels/tsa/statespace/mlemodel.py
--- a/statsmodels/tsa/statespace/mlemodel.py
+++ b/statsmodels/tsa/statespace/mlemodel.py
@@ -20,6 +20,7 @@
 )
 from statsmodels.tools.decorators import cache_readonly, resettable_cache
 from statsmodels.tools.eval_measures import aic, bic, hqic
+from statsmodels.tools.tools import pinv_extended
 
 
 class MLEModel(tsbase.TimeSeriesModel):
@@ -821,6 +822,7 @@ def __init__(self, model, params, filter_results, cov_type='opg',
         # Handle covariance matrix calculation
         if cov_kwds is None:
                 cov_kwds = {}
+        self._rank = None
         self._get_robustcov_results(cov_type=cov_type, use_self=True,
                                     **cov_kwds)
 
@@ -889,6 +891,7 @@ def _get_robustcov_results(self, cov_type='opg', **kwargs):
         # Calculate the new covariance matrix
         if len(self.params) == 0:
             res.cov_params_default = np.zeros((0,0))
+            res._rank = 0
             res.cov_kwds['cov_type'] = (
                 'No parameters estimated.')
         elif self.cov_type == 'cs':
@@ -962,7 +965,12 @@ def cov_params_cs(self):
         )
         self.model.update(self.params)
 
-        return -np.linalg.inv(nobs * evaluated_hessian)
+        neg_cov, singular_values = pinv_extended(nobs * evaluated_hessian)
+
+        if self._rank is None:
+            self._rank = np.linalg.matrix_rank(np.diag(singular_values))
+
+        return -neg_cov
 
     @cache_readonly
     def cov_params_delta(self):
@@ -975,11 +983,14 @@ def cov_params_delta(self):
 
         unconstrained = self.model.untransform_params(self.params)
         jacobian = self.model.transform_jacobian(unconstrained)
-        cov_cs = -np.linalg.inv(
+        neg_cov, singular_values = pinv_extended(
             nobs * self.model._hessian_cs(unconstrained, transformed=False)
         )
 
-        return np.dot(np.dot(jacobian, cov_cs), jacobian.transpose())
+        if self._rank is None:
+            self._rank = np.linalg.matrix_rank(np.diag(singular_values))
+
+        return np.dot(np.dot(jacobian, -neg_cov), jacobian.transpose())
 
     @cache_readonly
     def cov_params_oim(self):
@@ -988,10 +999,15 @@ def cov_params_oim(self):
         from Harvey (1989).
         """
         nobs = (self.model.nobs - self.filter_results.loglikelihood_burn)
-        return np.linalg.inv(
+        cov_params, singular_values = pinv_extended(
             nobs * self.model.observed_information_matrix(self.params)
         )
 
+        if self._rank is None:
+            self._rank = np.linalg.matrix_rank(np.diag(singular_values))
+
+        return cov_params
+
     @cache_readonly
     def cov_params_opg(self):
         """
@@ -1003,13 +1019,18 @@ def cov_params_opg(self):
         # because variance parameters will then have a complex component which
         # which implies non-positive-definiteness.
         inversion_method = INVERT_UNIVARIATE | SOLVE_LU
-        return np.linalg.inv(
+        cov_params, singular_values = pinv_extended(
             nobs *
             self.model.opg_information_matrix(
                 self.params, inversion_method=inversion_method
             )
         )
 
+        if self._rank is None:
+            self._rank = np.linalg.matrix_rank(np.diag(singular_values))
+
+        return cov_params
+
     @cache_readonly
     def cov_params_robust(self):
         """
@@ -1029,10 +1050,15 @@ def cov_params_robust_oim(self):
         evaluated_hessian = (
             nobs * self.model.observed_information_matrix(self.params)
         )
-        return np.linalg.inv(
+        cov_params, singular_values = pinv_extended(
             np.dot(np.dot(evaluated_hessian, cov_opg), evaluated_hessian)
         )
 
+        if self._rank is None:
+            self._rank = np.linalg.matrix_rank(np.diag(singular_values))
+
+        return cov_params
+
     @cache_readonly
     def cov_params_robust_cs(self):
         """
@@ -1051,10 +1077,15 @@ def cov_params_robust_cs(self):
             self.model._hessian_cs(self.params, transformed=True,
                                    inversion_method=inversion_method)
         )
-        return np.linalg.inv(
+        cov_params, singular_values = pinv_extended(
             np.dot(np.dot(evaluated_hessian, cov_opg), evaluated_hessian)
         )
 
+        if self._rank is None:
+            self._rank = np.linalg.matrix_rank(np.diag(singular_values))
+
+        return cov_params
+
     def fittedvalues(self):
         """
         (array) The predicted values of the model.
@@ -1281,6 +1312,10 @@ def summary(self, alpha=.05, start=None, model_name=None):
         etext = []
         if hasattr(self, 'cov_type') and 'description' in self.cov_kwds:
             etext.append(self.cov_kwds['description'])
+        if self._rank < len(self.params):
+            etext.append("Covariance matrix is singular or near-singular,"
+                         " with condition number %6.3g. Standard errors may be"
+                         " unstable." % np.linalg.cond(self.cov_params()))
 
         if etext:
             etext = ["[{0}] {1}".format(i + 1, text)


Test patch:
diff --git a/statsmodels/tsa/statespace/tests/test_representation.py b/statsmodels/tsa/statespace/tests/test_representation.py
--- a/statsmodels/tsa/statespace/tests/test_representation.py
+++ b/statsmodels/tsa/statespace/tests/test_representation.py
@@ -369,7 +369,7 @@ def __init__(self):
 
     def test_kalman_gain(self):
         assert_allclose(self.results.kalman_gain.sum(axis=1).sum(axis=0),
-                        clark1989_results['V1'], atol=1e-5)
+                        clark1989_results['V1'], atol=1e-4)
 
 
 class TestClark1989Conserve(Clark1989):