Repository: statsmodels/statsmodels

Issue:
auto lag selection in  S_hac_simple
the code (line 353 in sandwich_covariance.py 0.4.3) is

  nlags = np.floor[4 \* (n_periods / 100.)**(2./9.)]

shouldn't the [] be replaced with ()?



Discussion:
Thank you for catching this.

clearly a bug, and all tests and examples use a number for nlags. 

docstring has the same [], and Notes are outdated. The function has unit tests against Gretl and R for fixed nlags.

There is also a casting to integer missing. I get now:

```
DeprecationWarning: integer argument expected, got float
for lag in range(1, nlags+1):
```


Resolving patch:
diff --git a/statsmodels/stats/proportion.py b/statsmodels/stats/proportion.py
--- a/statsmodels/stats/proportion.py
+++ b/statsmodels/stats/proportion.py
@@ -248,7 +248,8 @@ def _power_ztost(mean_low, var_low, mean_upp, var_upp, mean_alt, var_alt,
     #print mean_low, np.sqrt(var_low), crit, var_low
     #print mean_upp, np.sqrt(var_upp), crit, var_upp
     if np.any(k_low > k_upp):   #vectorize
-        print "no overlap, power is zero, TODO"
+        import warnings
+        warnings.warn("no overlap, power is zero")
     std_alt = np.sqrt(var_alt)
     z_low = (k_low - mean_alt - continuity[0] * 0.5 / nobs) / std_alt
     z_upp = (k_upp - mean_alt + continuity[1] * 0.5 / nobs) / std_alt
diff --git a/statsmodels/stats/sandwich_covariance.py b/statsmodels/stats/sandwich_covariance.py
--- a/statsmodels/stats/sandwich_covariance.py
+++ b/statsmodels/stats/sandwich_covariance.py
@@ -326,7 +326,7 @@ def S_hac_simple(x, nlags=None, weights_func=weights_bartlett):
         data, for HAC this is array of x_i * u_i
     nlags : int or None
         highest lag to include in kernel window. If None, then
-        nlags = floor[4(T/100)^(2/9)] is used.
+        nlags = floor(4(T/100)^(2/9)) is used.
     weights_func : callable
         weights_func is called with nlags as argument to get the kernel
         weights. default are Bartlett weights
@@ -340,8 +340,6 @@ def S_hac_simple(x, nlags=None, weights_func=weights_bartlett):
     -----
     used by cov_hac_simple
 
-    verified only for nlags=0, which is just White, through cov_hac_simple
-
     options might change when other kernels besides Bartlett are available.
 
     '''
@@ -350,7 +348,7 @@ def S_hac_simple(x, nlags=None, weights_func=weights_bartlett):
         x = x[:,None]
     n_periods = x.shape[0]
     if nlags is None:
-        nlags = np.floor[4 * (n_periods / 100.)**(2./9.)]
+        nlags = int(np.floor(4 * (n_periods / 100.)**(2./9.)))
 
     weights = weights_func(nlags)
 


Test patch:
diff --git a/statsmodels/stats/tests/test_proportion.py b/statsmodels/stats/tests/test_proportion.py
--- a/statsmodels/stats/tests/test_proportion.py
+++ b/statsmodels/stats/tests/test_proportion.py
@@ -249,6 +249,7 @@ def test_binom_rejection_interval():
     pval = smprop.binom_test(ci_upp, nobs, prop=prop,
                                   alternative=alternative)
     assert_array_less(pval, alpha)
+
     pval = smprop.binom_test(ci_upp - 1, nobs, prop=prop,
                                   alternative=alternative)
     assert_array_less(alpha, pval)
@@ -301,35 +302,38 @@ def test_power_ztost_prop():
                          discrete=True, dist='binom')[0]
     assert_almost_equal(power, 0.8204, decimal=4) # PASS example
 
-    power = smprop.power_ztost_prop(0.4, 0.6, np.arange(20, 210, 20),
-                                    p_alt=0.5, alpha=0.05, discrete=False,
-                                    dist='binom')[0]
-
-    res_power = np.array([ 0., 0., 0., 0.0889, 0.2356, 0.4770, 0.5530,
-        0.6154,  0.7365,  0.7708])
-    # TODO: I currently don't impose power>=0, i.e np.maximum(power, 0)
-    assert_almost_equal(np.maximum(power, 0), res_power, decimal=4)
-
-    # with critval_continuity correction
-    power = smprop.power_ztost_prop(0.4, 0.6, np.arange(20, 210, 20),
-                                    p_alt=0.5, alpha=0.05, discrete=False,
-                                    dist='binom', variance_prop=None,
-                                    continuity=2, critval_continuity=1)[0]
-
-    res_power = np.array([0., 0., 0., 0.0889, 0.2356, 0.3517, 0.4457,
-                          0.6154, 0.6674, 0.7708])
-    # TODO: I currently don't impose power>=0, i.e np.maximum(power, 0)
-    assert_almost_equal(np.maximum(power, 0), res_power, decimal=4)
-
-    power = smprop.power_ztost_prop(0.4, 0.6, np.arange(20, 210, 20),
-                                    p_alt=0.5, alpha=0.05, discrete=False,
-                                    dist='binom', variance_prop=0.5,
-                                    critval_continuity=1)[0]
-
-    res_power = np.array([0., 0., 0., 0.0889, 0.2356, 0.3517, 0.4457,
-                          0.6154, 0.6674, 0.7112])
-    # TODO: I currently don't impose power>=0, i.e np.maximum(power, 0)
-    assert_almost_equal(np.maximum(power, 0), res_power, decimal=4)
+    import warnings
+    with warnings.catch_warnings():  # python >= 2.6
+        warnings.simplefilter("ignore")
+        power = smprop.power_ztost_prop(0.4, 0.6, np.arange(20, 210, 20),
+                                        p_alt=0.5, alpha=0.05, discrete=False,
+                                        dist='binom')[0]
+
+        res_power = np.array([ 0., 0., 0., 0.0889, 0.2356, 0.4770, 0.5530,
+            0.6154,  0.7365,  0.7708])
+        # TODO: I currently don't impose power>=0, i.e np.maximum(power, 0)
+        assert_almost_equal(np.maximum(power, 0), res_power, decimal=4)
+
+        # with critval_continuity correction
+        power = smprop.power_ztost_prop(0.4, 0.6, np.arange(20, 210, 20),
+                                        p_alt=0.5, alpha=0.05, discrete=False,
+                                        dist='binom', variance_prop=None,
+                                        continuity=2, critval_continuity=1)[0]
+
+        res_power = np.array([0., 0., 0., 0.0889, 0.2356, 0.3517, 0.4457,
+                              0.6154, 0.6674, 0.7708])
+        # TODO: I currently don't impose power>=0, i.e np.maximum(power, 0)
+        assert_almost_equal(np.maximum(power, 0), res_power, decimal=4)
+
+        power = smprop.power_ztost_prop(0.4, 0.6, np.arange(20, 210, 20),
+                                        p_alt=0.5, alpha=0.05, discrete=False,
+                                        dist='binom', variance_prop=0.5,
+                                        critval_continuity=1)[0]
+
+        res_power = np.array([0., 0., 0., 0.0889, 0.2356, 0.3517, 0.4457,
+                              0.6154, 0.6674, 0.7112])
+        # TODO: I currently don't impose power>=0, i.e np.maximum(power, 0)
+        assert_almost_equal(np.maximum(power, 0), res_power, decimal=4)
 
 def test_ztost():
     xfair = np.repeat([1,0], [228, 762-228])
diff --git a/statsmodels/stats/tests/test_sandwich.py b/statsmodels/stats/tests/test_sandwich.py
--- a/statsmodels/stats/tests/test_sandwich.py
+++ b/statsmodels/stats/tests/test_sandwich.py
@@ -84,6 +84,11 @@ def test_hac_simple():
     assert_almost_equal(cov1, cov1_r, decimal=14)
     assert_almost_equal(cov2, cov2_r, decimal=14)
 
+    # compare default for nlags
+    cov3 = sw.cov_hac_simple(res_olsg, use_correction=False)
+    cov4 = sw.cov_hac_simple(res_olsg, nlags=4, use_correction=False)
+    assert_almost_equal(cov3, cov4, decimal=14)
+
 if __name__ == '__main__':
     import nose
     nose.runmodule(argv=[__file__, '-vvs', '-x'], exit=False)
diff --git a/statsmodels/tsa/tests/test_ar.py b/statsmodels/tsa/tests/test_ar.py
--- a/statsmodels/tsa/tests/test_ar.py
+++ b/statsmodels/tsa/tests/test_ar.py
@@ -201,7 +201,7 @@ def test_dynamic_predict(self):
         # None, None
         start, end = None, None
         fv = res1.predict(dynamic=True)
-        assert_almost_equal(fv, res2.fcdyn[9:309], DECIMAL_4)
+        assert_allclose(fv, res2.fcdyn[9:309], rtol=7e-6)
 
 
 class TestAutolagAR(object):