Repository: statsmodels/statsmodels

Issue:
BUG: acorr_ljungbox for short time series is broken
`diag.acorr_ljungbox`  requires at least 41 observations with default lag, It raises an exception for shorter series.

http://stackoverflow.com/questions/39916771/error-in-acorr-ljungbox-from-statsmodel/39919347#39919347

```
>>> from statsmodels.stats import diagnostic as diag
>>> diag.acorr_ljungbox(np.random.random(50))[0].shape
(40,)
>>> diag.acorr_ljungbox(np.random.random(20), lags=5)
(array([ 0.36718151,  1.02009595,  1.23734092,  3.75338034,  4.35387236]), array([ 0.54454461,  0.60046677,  0.74406305,  0.44040973,  0.49966951]))
>>> diag.acorr_ljungbox(np.random.random(20))
Traceback (most recent call last):
  File "<pyshell#543>", line 1, in <module>
    diag.acorr_ljungbox(np.random.random(20))
  File "m:\josef_new\eclipse_ws\statsmodels\statsmodels_py34_pr\statsmodels\sandbox\stats\diagnostic.py", line 295, in acorr_ljungbox
    acf2norm = acfx[1:maxlag+1]**2 / (nobs - np.arange(1,maxlag+1))
ValueError: operands could not be broadcast together with shapes (19,) (40,) 
```

Solution: we need a cutoff as function of nobs, `lags = min((lags, nobs // 2)` or something like that.
`nobs - 1` or `nobs -5` or similar would be possible, but given that we return the results for all lags up to maxlags and the estimate won't be reliable with few observations for the long lags, I think nobs // 2 is a reasonable default. Users can always pick something else. 
(I don't have Stata open right now to check what they do for short time series.)

**update**
related: We have ljung-box duplicated, and seems to default to lags=min((nobs - 1, nlags) with nlags=40 by default

```
>>> from statsmodels.tsa import stattools as ts
>>> ts.acf(z, unbiased=False, qstat=True)[-1]
array([ 0.67152051,  0.89754045,  0.95671896,  0.59265665,  0.70013498,
        0.39934265,  0.50700587,  0.59843545,  0.60683743,  0.5344915 ,
        0.5145257 ,  0.56778241,  0.63762903,  0.70128111,  0.54131853,
        0.60332243,  0.66656227,  0.72635071,  0.77991177])
>>> diag.acorr_ljungbox(z, lags=19)[1]
array([ 0.67152051,  0.89754045,  0.95671896,  0.59265665,  0.70013498,
        0.39934265,  0.50700587,  0.59843545,  0.60683743,  0.5344915 ,
        0.5145257 ,  0.56778241,  0.63762903,  0.70128111,  0.54131853,
        0.60332243,  0.66656227,  0.72635071,  0.77991177])
```



Discussion:
here the fix: #3360 

Resolving patch:
diff --git a/statsmodels/sandbox/stats/diagnostic.py b/statsmodels/sandbox/stats/diagnostic.py
--- a/statsmodels/sandbox/stats/diagnostic.py
+++ b/statsmodels/sandbox/stats/diagnostic.py
@@ -227,7 +227,8 @@ def __call__(self, results_x, results_z):
 
 
 def acorr_ljungbox(x, lags=None, boxpierce=False):
-    '''Ljung-Box test for no autocorrelation
+    """
+    Ljung-Box test for no autocorrelation
 
     Parameters
     ----------
@@ -239,7 +240,7 @@ def acorr_ljungbox(x, lags=None, boxpierce=False):
         If lags is a list or array, then all lags are included up to the largest
         lag in the list, however only the tests for the lags in the list are
         reported.
-        If lags is None, then the default maxlag is 12*(nobs/100)^{1/4}
+        If lags is None, then the default maxlag is 'min((nobs // 2 - 2), 40)'
     boxpierce : {False, True}
         If true, then additional to the results of the Ljung-Box test also the
         Box-Pierce test results are returned
@@ -265,7 +266,7 @@ def acorr_ljungbox(x, lags=None, boxpierce=False):
     1d or nd ? axis ? ravel ?
     needs more testing
 
-    ''Verification''
+    *Verification*
 
     Looks correctly sized in Monte Carlo studies.
     not yet compared to verified values
@@ -278,22 +279,18 @@ def acorr_ljungbox(x, lags=None, boxpierce=False):
     ----------
     Greene
     Wikipedia
-
-    '''
+    """
     x = np.asarray(x)
     nobs = x.shape[0]
     if lags is None:
-        lags = lrange(1,41)  #TODO: check default; SS: changed to 40
+        lags = np.arange(1, min((nobs // 2 - 2), 40) + 1)
     elif isinstance(lags, (int, long)):
-        lags = lrange(1,lags+1)
-    maxlag = max(lags)
+        lags = np.arange(1, lags + 1)
     lags = np.asarray(lags)
-
+    maxlag = max(lags)
     acfx = acf(x, nlags=maxlag) # normalize by nobs not (nobs-nlags)
                              # SS: unbiased=False is default now
-#    acf2norm = acfx[1:maxlag+1]**2 / (nobs - np.arange(1,maxlag+1))
     acf2norm = acfx[1:maxlag+1]**2 / (nobs - np.arange(1,maxlag+1))
-
     qljungbox = nobs * (nobs+2) * np.cumsum(acf2norm)[lags-1]
     pval = stats.chi2.sf(qljungbox, lags)
     if not boxpierce:
@@ -303,6 +300,7 @@ def acorr_ljungbox(x, lags=None, boxpierce=False):
         pvalbp = stats.chi2.sf(qboxpierce, lags)
         return qljungbox, pval, qboxpierce, pvalbp
 
+
 def acorr_lm(x, maxlag=None, autolag='AIC', store=False, regresults=False):
     '''Lagrange Multiplier tests for autocorrelation
 


Test patch:
diff --git a/statsmodels/stats/tests/test_diagnostic.py b/statsmodels/stats/tests/test_diagnostic.py
--- a/statsmodels/stats/tests/test_diagnostic.py
+++ b/statsmodels/stats/tests/test_diagnostic.py
@@ -307,8 +307,26 @@ def test_acorr_breusch_godfrey(self):
         assert_almost_equal(bg2, bg3, decimal=13)
 
     def test_acorr_ljung_box(self):
+
+        #unit-test which may be useful later
+        #ddof correction for fitted parameters in ARMA(p,q) fitdf=p+q
+        #> bt = Box.test(residuals(fm), lag=4, type = "Ljung-Box", fitdf=2)
+        #> mkhtest(bt, "ljung_box_4df2", "chi2")
+        # ljung_box_4df2 = dict(statistic=5.23587172795227,
+        #                       pvalue=0.0729532930400377,
+        #                       parameters=(2,), distr='chi2')
+
+        #> bt = Box.test(residuals(fm), lag=4, type = "Box-Pierce", fitdf=2)
+        #> mkhtest(bt, "ljung_box_bp_4df2", "chi2")
+        # ljung_box_bp_4df2 = dict(statistic=5.12462932741681,
+        #                          pvalue=0.0771260128929921,
+        #                          parameters=(2,), distr='chi2')
+
+
         res = self.res
 
+        #general test
+        
         #> bt = Box.test(residuals(fm), lag=4, type = "Ljung-Box")
         #> mkhtest(bt, "ljung_box_4", "chi2")
         ljung_box_4 = dict(statistic=5.23587172795227, pvalue=0.263940335284713,
@@ -320,24 +338,48 @@ def test_acorr_ljung_box(self):
                               pvalue=0.2747471266820692,
                               parameters=(4,), distr='chi2')
 
-        #ddof correction for fitted parameters in ARMA(p,q) fitdf=p+q
-        #> bt = Box.test(residuals(fm), lag=4, type = "Ljung-Box", fitdf=2)
-        #> mkhtest(bt, "ljung_box_4df2", "chi2")
-        ljung_box_4df2 = dict(statistic=5.23587172795227,
-                              pvalue=0.0729532930400377,
-                              parameters=(2,), distr='chi2')
-
-        #> bt = Box.test(residuals(fm), lag=4, type = "Box-Pierce", fitdf=2)
-        #> mkhtest(bt, "ljung_box_bp_4df2", "chi2")
-        ljung_box_bp_4df2 = dict(statistic=5.12462932741681,
-                                 pvalue=0.0771260128929921,
-                                 parameters=(2,), distr='chi2')
-
 
         lb, lbpval, bp, bppval = smsdia.acorr_ljungbox(res.resid, 4,
                                                        boxpierce=True)
-        compare_t_est([lb[-1], lbpval[-1]], ljung_box_4, decimal=(13, 14))
-        compare_t_est([bp[-1], bppval[-1]], ljung_box_bp_4, decimal=(13, 14))
+        compare_t_est([lb[-1], lbpval[-1]], ljung_box_4, decimal=(13, 13))
+        compare_t_est([bp[-1], bppval[-1]], ljung_box_bp_4, decimal=(13, 13))
+
+    def test_acorr_ljung_box_big_default(self):
+        res = self.res
+        #test with big dataset and default lag
+
+        #> bt = Box.test(residuals(fm), type = "Ljung-Box")
+        #> mkhtest(bt, "ljung_box_none", "chi2")
+        ljung_box_none = dict(statistic=51.03724531797195, pvalue=0.11334744923390,
+                              distr='chi2')
+
+        #> bt = Box.test(residuals(fm), type = "Box-Pierce")
+        #> mkhtest(bt, "ljung_box_bp_none", "chi2")
+        ljung_box_bp_none = dict(statistic=45.12238537034000,
+                              pvalue=0.26638168491464,
+                              distr='chi2')
+        lb, lbpval, bp, bppval = smsdia.acorr_ljungbox(res.resid, boxpierce=True)
+        compare_t_est([lb[-1], lbpval[-1]], ljung_box_none, decimal=(13, 13))
+        compare_t_est([bp[-1], bppval[-1]], ljung_box_bp_none, decimal=(13, 13))
+
+    def test_acorr_ljung_box_small_default(self):
+        res = self.res
+        #test with small dataset and default lag
+        
+        #> bt = Box.test(residuals(fm), type = "Ljung-Box")
+        #> mkhtest(bt, "ljung_box_small", "chi2")
+        ljung_box_small = dict(statistic=9.61503968281915, pvalue=0.72507000996945,
+                           parameters=(0,), distr='chi2')
+
+        #> bt = Box.test(residuals(fm), type = "Box-Pierce")
+        #> mkhtest(bt, "ljung_box_bp_small", "chi2")
+        ljung_box_bp_small = dict(statistic=7.41692150864936,
+                              pvalue=0.87940785887006,
+                              parameters=(0,), distr='chi2')
+
+        lb, lbpval, bp, bppval = smsdia.acorr_ljungbox(res.resid[:30], boxpierce=True)
+        compare_t_est([lb[-1], lbpval[-1]], ljung_box_small, decimal=(13, 13))
+        compare_t_est([bp[-1], bppval[-1]], ljung_box_bp_small, decimal=(13, 13))
 
 
     def test_harvey_collier(self):