Repository: statsmodels/statsmodels

Issue:
O(T^2) complexity of stattools.acf
Hi, 
Computing the acf of a vector of length T=10^6 (with nlags=100) takes more than 10 minutes on my machine. In addition, it seems impossible to stop the execution before completion (using Ctrl-C). 

Looking at the code, I think the problem comes from this line:
`>>  acov = (np.correlate(xo, xo, 'full') / d)[n - 1:]`
in function acovf

Correct me if I'm wrong, but this seems to compute the ACF for *all* the lags 1, ..., T-1, then keeping only the (n-1) first. This means the complexity is O(T^2), whereas computing the ACF at lags 1 to L 
should have complexity O(TL). In most practical cases L << T.

Looking at the doc, I can see now that with the fft option I can compute the ACF faster. However, I think 
it's problematic that the default implementation would be so inefficient. The acf command in R does not have such an option, and manages to compute and plot the ACF of a vector of length T=10^6 in one second. 

Thank you for attention
Nicolas




Discussion:
use `fft=True`
I think most examples that we had in mind for acf will not have T >> L. We use explicit loops over lags in some other cases where we expect L to be small.

I didn't follow the related recent discussions closely and don't have an overview anymore:
numpy PR to add maxlag option to correlate has stalled https://github.com/numpy/numpy/pull/5978 https://github.com/scipy/scipy/issues/4940 https://mail.python.org/pipermail/numpy-discussion/2016-May/075398.html

In scipy.signal there was also the discussion to combine convolve and fft_convolve with auto option, but I didn't pay attention.

Besides using an explicit loop, it is also possible to adjust the correlate options and padding to limit maxlag, https://github.com/NeuralEnsemble/elephant/blob/master/elephant/spike_train_correlation.py#L535 (link was provided in numpy PR)



Never underestimate the ability of users to use your software in crazy ways... :-) 
More seriously, I was trying to compute the ACF of MCMC chains
https://en.wikipedia.org/wiki/Markov_chain_Monte_Carlo
as reporting the ACF is standard practice in MCMC simulation.  In MCMC it is typical to have T = 10^6 - 10^8, and a number of lags L << T. So other MCMC ppl might run into the same issue. (plus also maybe ppl dealing with very long time series like in e.g. finance.) 

I was unaware that this was related to some numpy issues: looping over l, as josef-pkt suggests seems quite reasonable to me. At any rate, thanks for looking into this. 
 

Resolving patch:
diff --git a/statsmodels/sandbox/stats/diagnostic.py b/statsmodels/sandbox/stats/diagnostic.py
--- a/statsmodels/sandbox/stats/diagnostic.py
+++ b/statsmodels/sandbox/stats/diagnostic.py
@@ -288,8 +288,9 @@ def acorr_ljungbox(x, lags=None, boxpierce=False):
         lags = np.arange(1, lags + 1)
     lags = np.asarray(lags)
     maxlag = max(lags)
-    acfx = acf(x, nlags=maxlag) # normalize by nobs not (nobs-nlags)
-                                # SS: unbiased=False is default now
+    # normalize by nobs not (nobs-nlags)
+    # SS: unbiased=False is default now
+    acfx = acf(x, nlags=maxlag, fft=False)
     acf2norm = acfx[1:maxlag+1]**2 / (nobs - np.arange(1,maxlag+1))
     qljungbox = nobs * (nobs+2) * np.cumsum(acf2norm)[lags-1]
     pval = stats.chi2.sf(qljungbox, lags)
diff --git a/statsmodels/tsa/stattools.py b/statsmodels/tsa/stattools.py
--- a/statsmodels/tsa/stattools.py
+++ b/statsmodels/tsa/stattools.py
@@ -293,7 +293,7 @@ def adfuller(x, maxlag=None, regression="c", autolag='AIC',
             return adfstat, pvalue, usedlag, nobs, critvalues, icbest
 
 
-def acovf(x, unbiased=False, demean=True, fft=False, missing='none'):
+def acovf(x, unbiased=False, demean=True, fft=None, missing='none', nlag=None):
     """
     Autocovariance for 1D
 
@@ -309,8 +309,14 @@ def acovf(x, unbiased=False, demean=True, fft=False, missing='none'):
         If True, use FFT convolution.  This method should be preferred
         for long time series.
     missing : str
-        A string in ['none', 'raise', 'conservative', 'drop'] specifying how the NaNs
-        are to be treated.
+        A string in ['none', 'raise', 'conservative', 'drop'] specifying how
+        the NaNs are to be treated.
+    nlag : {int, None}
+        Limit the number of autocovariances returned.  Size of returned
+        array is nlag + 1.  Setting nlag when fft is False uses a simple,
+        direct estimator of the autocovariances that only computes the first
+        nlag + 1 values. This can be much faster when the time series is long
+        and only a small number of autocovariances are needed.
 
     Returns
     -------
@@ -323,6 +329,14 @@ def acovf(x, unbiased=False, demean=True, fft=False, missing='none'):
            and amplitude modulation. Sankhya: The Indian Journal of
            Statistics, Series A, pp.383-392.
     """
+    if fft is None:
+        import warnings
+        msg = 'fft=True will become the default in a future version of ' \
+              'statsmodels. To suppress this warning, explicitly set ' \
+              'fft=False.'
+        warnings.warn(msg, FutureWarning)
+        fft = False
+
     x = np.squeeze(np.asarray(x))
     if x.ndim > 1:
         raise ValueError("x must be 1d. Got %d dims." % x.ndim)
@@ -337,17 +351,19 @@ def acovf(x, unbiased=False, demean=True, fft=False, missing='none'):
     if deal_with_masked:
         if missing == 'raise':
             raise MissingDataError("NaNs were encountered in the data")
-        notmask_bool = ~np.isnan(x) #bool
+        notmask_bool = ~np.isnan(x)  # bool
         if missing == 'conservative':
+            # Must copy for thread safety
+            x = x.copy()
             x[~notmask_bool] = 0
-        else: #'drop'
-            x = x[notmask_bool] #copies non-missing
-        notmask_int = notmask_bool.astype(int) #int
+        else:  # 'drop'
+            x = x[notmask_bool]  # copies non-missing
+        notmask_int = notmask_bool.astype(int)  # int
 
     if demean and deal_with_masked:
         # whether 'drop' or 'conservative':
-        xo = x - x.sum()/notmask_int.sum()
-        if missing=='conservative':
+        xo = x - x.sum() / notmask_int.sum()
+        if missing == 'conservative':
             xo[~notmask_bool] = 0
     elif demean:
         xo = x - x.mean()
@@ -355,14 +371,43 @@ def acovf(x, unbiased=False, demean=True, fft=False, missing='none'):
         xo = x
 
     n = len(x)
-    if unbiased and deal_with_masked and missing=='conservative':
+    lag_len = nlag
+    if nlag is None:
+        lag_len = n - 1
+    elif nlag > n - 1:
+        raise ValueError('nlag must be smaller than nobs - 1')
+
+    if not fft and nlag is not None:
+        acov = np.empty(lag_len + 1)
+        acov[0] = xo.dot(xo)
+        for i in range(lag_len):
+            acov[i + 1] = xo[i + 1:].dot(xo[:-(i + 1)])
+        if not deal_with_masked or missing == 'drop':
+            if unbiased:
+                acov /= (n - np.arange(lag_len + 1))
+            else:
+                acov /= n
+        else:
+            if unbiased:
+                divisor = np.empty(lag_len + 1, dtype=np.int64)
+                divisor[0] = notmask_int.sum()
+                for i in range(lag_len):
+                    divisor[i + 1] = notmask_int[i + 1:].dot(notmask_int[:-(i + 1)])
+                divisor[divisor == 0] = 1
+                acov /= divisor
+            else:  # biased, missing data but npt 'drop'
+                acov /= notmask_int.sum()
+        return acov
+
+    if unbiased and deal_with_masked and missing == 'conservative':
         d = np.correlate(notmask_int, notmask_int, 'full')
+        d[d == 0] = 1
     elif unbiased:
         xi = np.arange(1, n + 1)
         d = np.hstack((xi, xi[:-1][::-1]))
-    elif deal_with_masked: #biased and NaNs given and ('drop' or 'conservative')
-        d = notmask_int.sum() * np.ones(2*n-1)
-    else: #biased and no NaNs or missing=='none'
+    elif deal_with_masked:  # biased and NaNs given and ('drop' or 'conservative')
+        d = notmask_int.sum() * np.ones(2 * n - 1)
+    else:  # biased and no NaNs or missing=='none'
         d = n * np.ones(2 * n - 1)
 
     if fft:
@@ -372,12 +417,11 @@ def acovf(x, unbiased=False, demean=True, fft=False, missing='none'):
         acov = np.fft.ifft(Frf * np.conjugate(Frf))[:nobs] / d[nobs - 1:]
         acov = acov.real
     else:
-        acov = (np.correlate(xo, xo, 'full') / d)[n - 1:]
-
-    if deal_with_masked and missing=='conservative':
-        # restore data for the user
-        x[~notmask_bool] = np.nan
+        acov = np.correlate(xo, xo, 'full')[n - 1:] / d[n - 1:]
 
+    if nlag is not None:
+        # Copy to allow gc of full array rather than view
+        return acov[:lag_len + 1].copy()
     return acov
 
 
@@ -413,7 +457,7 @@ def q_stat(x, nobs, type="ljungbox"):
 #NOTE: Changed unbiased to False
 #see for example
 # http://www.itl.nist.gov/div898/handbook/eda/section3/autocopl.htm
-def acf(x, unbiased=False, nlags=40, qstat=False, fft=False, alpha=None,
+def acf(x, unbiased=False, nlags=40, qstat=False, fft=None, alpha=None,
         missing='none'):
     """
     Autocorrelation function for 1d arrays.
@@ -456,11 +500,14 @@ def acf(x, unbiased=False, nlags=40, qstat=False, fft=False, alpha=None,
     -----
     The acf at lag 0 (ie., 1) is returned.
 
-    This is based np.correlate which does full convolution. For very long time
-    series it is recommended to use fft convolution instead.
+    For very long time series it is recommended to use fft convolution instead.
+    When fft is False uses a simple, direct estimator of the autocovariances
+    that only computes the first nlag + 1 values. This can be much faster when
+    the time series is long and only a small number of autocovariances are
+    needed.
 
     If unbiased is true, the denominator for the autocovariance is adjusted
-    but the autocorrelation is not an unbiased estimtor.
+    but the autocorrelation is not an unbiased estimator.
 
     References
     ----------
@@ -469,6 +516,14 @@ def acf(x, unbiased=False, nlags=40, qstat=False, fft=False, alpha=None,
        Statistics, Series A, pp.383-392.
 
     """
+    if fft is None:
+        import warnings
+        msg = 'fft=True will become the default in a future version of ' \
+              'statsmodels. To suppress this warning, explicitly set ' \
+              'fft=False.'
+        warnings.warn(msg, FutureWarning)
+        fft = False
+
     nobs = len(x)  # should this shrink for missing='drop' and NaNs in x?
     avf = acovf(x, unbiased=unbiased, demean=True, fft=fft, missing=missing)
     acf = avf[:nlags + 1] / avf[0]
@@ -600,13 +655,13 @@ def pacf(x, nlags=40, method='ywunbiased', alpha=None):
     elif method in ['ywm', 'ywmle', 'yw_mle']:
         ret = pacf_yw(x, nlags=nlags, method='mle')
     elif method in ['ld', 'ldu', 'ldunbiase', 'ld_unbiased']:
-        acv = acovf(x, unbiased=True)
+        acv = acovf(x, unbiased=True, fft=False)
         ld_ = levinson_durbin(acv, nlags=nlags, isacov=True)
         #print 'ld', ld_
         ret = ld_[2]
     # inconsistent naming with ywmle
     elif method in ['ldb', 'ldbiased', 'ld_biased']:
-        acv = acovf(x, unbiased=False)
+        acv = acovf(x, unbiased=False, fft=False)
         ld_ = levinson_durbin(acv, nlags=nlags, isacov=True)
         ret = ld_[2]
     else:
@@ -759,7 +814,7 @@ def levinson_durbin(s, nlags=10, isacov=False):
     if isacov:
         sxx_m = s
     else:
-        sxx_m = acovf(s)[:order + 1]  # not tested
+        sxx_m = acovf(s, fft=False)[:order + 1]  # not tested
 
     phi = np.zeros((order + 1, order + 1), 'd')
     sig = np.zeros(order + 1)


Test patch:
diff --git a/statsmodels/tsa/statespace/tests/test_tools.py b/statsmodels/tsa/statespace/tests/test_tools.py
--- a/statsmodels/tsa/statespace/tests/test_tools.py
+++ b/statsmodels/tsa/statespace/tests/test_tools.py
@@ -351,7 +351,7 @@ def test_multivariate_acovf():
     x = np.arange(20)*1.0
     assert_allclose(
         np.squeeze(tools._compute_multivariate_sample_acovf(x, maxlag=4)),
-        acovf(x)[:5])
+        acovf(x, fft=False)[:5])
 
 
 def test_multivariate_pacf():
diff --git a/statsmodels/tsa/tests/test_stattools.py b/statsmodels/tsa/tests/test_stattools.py
--- a/statsmodels/tsa/tests/test_stattools.py
+++ b/statsmodels/tsa/tests/test_stattools.py
@@ -1,3 +1,5 @@
+from statsmodels.compat.python import lrange, PY3
+
 import os
 import warnings
 
@@ -8,8 +10,6 @@
                            assert_, assert_allclose)
 from pandas import Series, DatetimeIndex, DataFrame
 
-from statsmodels.compat.numpy import recarray_select
-from statsmodels.compat.python import lrange
 from statsmodels.datasets import macrodata, sunspots
 from statsmodels.tools.sm_exceptions import (CollinearityWarning,
                                              MissingDataError)
@@ -28,6 +28,12 @@
 DECIMAL_1 = 1
 
 
+@pytest.fixture('module')
+def acovf_data():
+    rnd = np.random.RandomState(12345)
+    return rnd.randn(250)
+
+
 class CheckADF(object):
     """
     Test Augmented Dickey-Fuller
@@ -158,7 +164,7 @@ def setup_class(cls):
         cls.acf = cls.results['acvar']
         #cls.acf = np.concatenate(([1.], cls.acf))
         cls.qstat = cls.results['Q1']
-        cls.res1 = acf(cls.x, nlags=40, qstat=True, alpha=.05)
+        cls.res1 = acf(cls.x, nlags=40, qstat=True, alpha=.05, fft=False)
         cls.confint_res = cls.results[['acvar_lb','acvar_ub']].values
 
     def test_acf(self):
@@ -201,17 +207,18 @@ def setup_class(cls):
         cls.acf = cls.results['acvar'] # drop and conservative
         cls.qstat = cls.results['Q1']
         cls.res_drop = acf(cls.x, nlags=40, qstat=True, alpha=.05,
-                            missing='drop')
+                           missing='drop', fft=False)
         cls.res_conservative = acf(cls.x, nlags=40, qstat=True, alpha=.05,
-                                    missing='conservative')
-        cls.acf_none = np.empty(40) * np.nan # lags 1 to 40 inclusive
+                                   fft=False, missing='conservative')
+        cls.acf_none = np.empty(40) * np.nan  # lags 1 to 40 inclusive
         cls.qstat_none = np.empty(40) * np.nan
         cls.res_none = acf(cls.x, nlags=40, qstat=True, alpha=.05,
-                        missing='none')
+                           missing='none', fft=False)
 
     def test_raise(self):
-        assert_raises(MissingDataError, acf, self.x, nlags=40,
-                      qstat=True, alpha=.05, missing='raise')
+        with pytest.raises(MissingDataError):
+            acf(self.x, nlags=40, qstat=True, fft=False, alpha=.05,
+                missing='raise')
 
     def test_acf_none(self):
         assert_almost_equal(self.res_none[0][1:41], self.acf_none, DECIMAL_8)
@@ -489,17 +496,18 @@ def test_lags(self):
 
 def test_pandasacovf():
     s = Series(lrange(1, 11))
-    assert_almost_equal(acovf(s), acovf(s.values))
+    assert_almost_equal(acovf(s, fft=False), acovf(s.values, fft=False))
 
 
 def test_acovf2d():
     dta = sunspots.load_pandas().data
     dta.index = DatetimeIndex(start='1700', end='2009', freq='A')[:309]
     del dta["YEAR"]
-    res = acovf(dta)
-    assert_equal(res, acovf(dta.values))
-    X = np.random.random((10,2))
-    assert_raises(ValueError, acovf, X)
+    res = acovf(dta, fft=False)
+    assert_equal(res, acovf(dta.values, fft=False))
+    x = np.random.random((10, 2))
+    with pytest.raises(ValueError):
+        acovf(x, fft=False)
 
 
 def test_acovf_fft_vs_convolution():
@@ -602,6 +610,42 @@ def test_levinson_durbin_acov():
     assert_allclose(pacf, np.array([1, rho] + [0] * (m - 1)), atol=1e-8)
 
 
-if __name__=="__main__":
-    import pytest
-    pytest.main([__file__, '-vvs', '-x', '--pdb'])
+@pytest.mark.parametrize("missing", ['conservative', 'drop', 'raise', 'none'])
+@pytest.mark.parametrize("fft", [False, True])
+@pytest.mark.parametrize("demean", [True, False])
+@pytest.mark.parametrize("unbiased", [True, False])
+def test_acovf_nlags(acovf_data, unbiased, demean, fft, missing):
+    full = acovf(acovf_data, unbiased=unbiased, demean=demean, fft=fft,
+                 missing=missing)
+    limited = acovf(acovf_data, unbiased=unbiased, demean=demean, fft=fft,
+                    missing=missing, nlag=10)
+    assert_allclose(full[:11], limited)
+
+
+@pytest.mark.parametrize("missing", ['conservative', 'drop'])
+@pytest.mark.parametrize("fft", [False, True])
+@pytest.mark.parametrize("demean", [True, False])
+@pytest.mark.parametrize("unbiased", [True, False])
+def test_acovf_nlags_missing(acovf_data, unbiased, demean, fft, missing):
+    acovf_data = acovf_data.copy()
+    acovf_data[1:3] = np.nan
+    full = acovf(acovf_data, unbiased=unbiased, demean=demean, fft=fft,
+                 missing=missing)
+    limited = acovf(acovf_data, unbiased=unbiased, demean=demean, fft=fft,
+                    missing=missing, nlag=10)
+    assert_allclose(full[:11], limited)
+
+
+def test_acovf_error(acovf_data):
+    with pytest.raises(ValueError):
+        acovf(acovf_data, nlag=250, fft=False)
+
+
+def test_acovf_warns(acovf_data):
+    with pytest.warns(FutureWarning):
+        acovf(acovf_data)
+
+
+def test_acf_warns(acovf_data):
+    with pytest.warns(FutureWarning):
+        acf(acovf_data, nlags=40)
diff --git a/statsmodels/tsa/tests/test_tsa_tools.py b/statsmodels/tsa/tests/test_tsa_tools.py
--- a/statsmodels/tsa/tests/test_tsa_tools.py
+++ b/statsmodels/tsa/tests/test_tsa_tools.py
@@ -24,9 +24,9 @@
 
 
 def test_acf():
-    acf_x = tsa.acf(x100, unbiased=False)[:21]
+    acf_x = tsa.acf(x100, unbiased=False, fft=False)[:21]
     assert_array_almost_equal(mlacf.acf100.ravel(), acf_x, 8)  # why only dec=8
-    acf_x = tsa.acf(x1000, unbiased=False)[:21]
+    acf_x = tsa.acf(x1000, unbiased=False, fft=False)[:21]
     assert_array_almost_equal(mlacf.acf1000.ravel(), acf_x, 8)  # why only dec=9