Repository: statsmodels/statsmodels

Issue:
KPSS autolag (issue2781)
KPSS autolag method of Hobijn et al. (1998). For more info see issue2781: https://github.com/statsmodels/statsmodels/issues/2781


Discussion:

Resolving patch:
diff --git a/statsmodels/tsa/stattools.py b/statsmodels/tsa/stattools.py
--- a/statsmodels/tsa/stattools.py
+++ b/statsmodels/tsa/stattools.py
@@ -1580,9 +1580,12 @@ def kpss(x, regression='c', lags=None, store=False):
         Indicates the null hypothesis for the KPSS test
         * 'c' : The data is stationary around a constant (default)
         * 'ct' : The data is stationary around a trend
-    lags : int
-        Indicates the number of lags to be used. If None (default),
-        lags is set to int(12 * (n / 100)**(1 / 4)), as outlined in
+    lags : {None, str, int}, optional
+        Indicates the number of lags to be used. If None (default), lags is
+        calculated using the legacy method. If 'auto', lags is calculated
+        using the data-dependent method of Hobijn et al. (1998). See also
+        Andrews (1991), Newey & West (1994), and Schwert (1989). If set to
+        'legacy',  uses int(12 * (n / 100)**(1 / 4)) , as outlined in
         Schwert (1989).
     store : bool
         If True, then a result instance is returned additionally to
@@ -1619,9 +1622,21 @@ def kpss(x, regression='c', lags=None, store=False):
 
     References
     ----------
-    D. Kwiatkowski, P. C. B. Phillips, P. Schmidt, and Y. Shin (1992): Testing
-    the Null Hypothesis of Stationarity against the Alternative of a Unit Root.
-    `Journal of Econometrics` 54, 159-178.
+    Andrews, D.W.K. (1991). Heteroskedasticity and autocorrelation consistent
+    covariance matrix estimation. Econometrica, 59: 817-858.
+
+    Hobijn, B., Frances, B.H., & Ooms, M. (2004). Generalizations of the
+    KPSS-test for stationarity. Statistica Neerlandica, 52: 483-502.
+
+    Kwiatkowski, D., Phillips, P.C.B., Schmidt, P., & Shin, Y. (1992). Testing
+    the null hypothesis of stationarity against the alternative of a unit root.
+    Journal of Econometrics, 54: 159-178.
+
+    Newey, W.K., & West, K.D. (1994). Automatic lag selection in covariance
+    matrix estimation. Review of Economic Studies, 61: 631-653.
+
+    Schwert, G. W. (1989). Tests for unit roots: A Monte Carlo investigation.
+    Journal of Business and Economic Statistics, 7 (2): 147-159.
     """
     from warnings import warn
 
@@ -1648,8 +1663,23 @@ def kpss(x, regression='c', lags=None, store=False):
         raise ValueError("hypothesis '{0}' not understood".format(hypo))
 
     if lags is None:
-        # from Kwiatkowski et al. referencing Schwert (1989)
+        lags = 'legacy'
+        msg = 'The behavior of using lags=None will change in the next ' \
+              'release. Currently lags=None is the same as ' \
+              'lags=\'legacy\', and so a sample-size lag length is used. ' \
+              'After the next release, the default will change to be the ' \
+              'same as lags=\'auto\' which uses an automatic lag length ' \
+              'selection method. To silence this warning, either use ' \
+              '\'auto\' or \'legacy\''
+        import warnings
+        warnings.warn(msg, DeprecationWarning)
+    if lags == 'legacy':
         lags = int(np.ceil(12. * np.power(nobs / 100., 1 / 4.)))
+    elif lags == 'auto':
+        # autolag method of Hobijn et al. (1998)
+        lags = _kpss_autolag(resids, nobs)
+    else:
+        lags = int(lags)
 
     pvals = [0.10, 0.05, 0.025, 0.01]
 
@@ -1690,3 +1720,24 @@ def _sigma_est_kpss(resids, nobs, lags):
         resids_prod = np.dot(resids[i:], resids[:nobs - i])
         s_hat += 2 * resids_prod * (1. - (i / (lags + 1.)))
     return s_hat / nobs
+
+
+def _kpss_autolag(resids, nobs):
+    """
+    Computes the number of lags for covariance matrix estimation in KPSS test
+    using method of Hobijn et al (1998). See also Andrews (1991), Newey & West
+    (1994), and Schwert (1989). Assumes Bartlett / Newey-West kernel.
+    """
+    covlags = int(np.power(nobs, 2. / 9.))
+    s0 = sum(resids**2) / nobs
+    s1 = 0
+    for i in range(1, covlags + 1):
+        resids_prod = np.dot(resids[i:], resids[:nobs - i])
+        resids_prod /= (nobs / 2.)
+        s0 += resids_prod
+        s1 += i * resids_prod
+    s_hat = s1 / s0
+    pwr = 1. / 3.
+    gamma_hat = 1.1447 * np.power(s_hat * s_hat, pwr)
+    autolags = np.amin([nobs, int(gamma_hat * np.power(nobs, pwr))])
+    return autolags


Test patch:
diff --git a/statsmodels/tsa/tests/test_stattools.py b/statsmodels/tsa/tests/test_stattools.py
--- a/statsmodels/tsa/tests/test_stattools.py
+++ b/statsmodels/tsa/tests/test_stattools.py
@@ -13,7 +13,7 @@
                            assert_, assert_allclose)
 from pandas import Series, date_range, DataFrame
 
-from statsmodels.datasets import macrodata, sunspots
+from statsmodels.datasets import macrodata, sunspots, nile, randhie, modechoice
 from statsmodels.tools.sm_exceptions import (CollinearityWarning,
                                              MissingDataError)
 from statsmodels.tsa.stattools import (adfuller, acf, pacf_yw, pacf_ols,
@@ -511,11 +511,44 @@ def test_store(self):
         assert_equal(store.nobs, len(self.x))
         assert_equal(store.lags, 3)
 
+    # test autolag function _kpss_autolag against SAS 9.3
     def test_lags(self):
-        with warnings.catch_warnings(record=True) as w:
-            kpss_stat, pval, lags, crits = kpss(self.x, 'c')
-        assert_equal(lags, int(np.ceil(12. * np.power(len(self.x) / 100., 1 / 4.))))
-        # assert_warns(UserWarning, kpss, self.x)
+        # real GDP from macrodata data set
+        with warnings.catch_warnings(record=True):
+            lags = kpss(self.x, 'c', lags='auto')[2]
+        assert_equal(lags, 9)
+        # real interest rates from macrodata data set
+        with warnings.catch_warnings(record=True):
+            lags = kpss(sunspots.load().data['SUNACTIVITY'], 'c',
+                        lags='auto')[2]
+        assert_equal(lags, 7)
+        # volumes from nile data set
+        with warnings.catch_warnings(record=True):
+            lags = kpss(nile.load().data['volume'], 'c', lags='auto')[2]
+        assert_equal(lags, 5)
+        # log-coinsurance from randhie data set
+        with warnings.catch_warnings(record=True):
+            lags = kpss(randhie.load().data['lncoins'], 'ct', lags='auto')[2]
+        assert_equal(lags, 75)
+        # in-vehicle time from modechoice data set
+        with warnings.catch_warnings(record=True):
+            lags = kpss(modechoice.load().data['invt'], 'ct', lags='auto')[2]
+        assert_equal(lags, 18)
+
+    def test_legacy_lags(self):
+        # Test legacy lags are the same
+        with warnings.catch_warnings(record=True):
+            lags = kpss(self.x, 'c', lags='legacy')[2]
+        assert_equal(lags, 15)
+
+    def test_unknown_lags(self):
+        # Test legacy lags are the same
+        with pytest.raises(ValueError):
+            kpss(self.x, 'c', lags='unknown')
+
+    def test_deprecation(self):
+        with pytest.deprecated_call():
+            kpss(self.x, 'c', lags=None)
 
 
 def test_pandasacovf():