Repository: statsmodels/statsmodels

Issue:
BUG: etest_poisson_2indep() not allowing custom ygrid
[The documentation](https://www.statsmodels.org/dev/generated/statsmodels.stats.rates.etest_poisson_2indep.html#statsmodels.stats.rates.etest_poisson_2indep) states that `ygrid` is a parameter we can supply to `etest_poisson_2indep`, but this is broken at the moment:

```python
from statsmodels.stats.rates import etest_poisson_2indep
etest_poisson_2indep(1, 1, 1, 1, ygrid=1)
```

Output:
```bash
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/home/voditas/.local/lib/python3.7/site-packages/statsmodels/stats/rates.py", line 249, in etest_poisson_2indep
    pdf1 = stats.poisson.pmf(y_grid, mean1)
UnboundLocalError: local variable 'y_grid' referenced before assignment
```

This happens in version 0.13.2 (but probably others too) for any value of `ygrid` that is not `None` and is because of the likely accidental usage of `y_grid` instead of `ygrid` in the function:

```python
  if ygrid is None:                                      # uses ygrid, like the API
        threshold = stats.poisson.isf(1e-13, max(mean1, mean2))
        threshold = max(threshold, 100)   # keep at least 100
        y_grid = np.arange(threshold + 1)                # changes to y_grid
pdf1 = stats.poisson.pmf(y_grid, mean1)                  # if ygrid was set, an error will be thrown
pdf2 = stats.poisson.pmf(y_grid, mean2)
```

The fix is easy - replace usages of `y_grid` with `ygrid`, as in the API. I will create a PR with this fix.



Discussion:
thanks for catching this

you could just do y_grid = ygrid in a new `else` part.

```
if ygrid is None:  
   ...
else:
   <needs some checking and asarray>
   y_grid = ... ygrid
```

I thought of doing that just to make less code changes, but I think it's confusing to have both `ygrid` and `y_grid`. There is no reason to make a distinction, is there?
It's taste question when there is no convention yet, and I'm changing my pattern every once in a while. 
Our code convention has some preference for adding underlines, but not strict in borderline cases.

I like `y_grid`
Recently I added a `y_values` option to count model predict (which doesn't need to be a grid). 

So, I guess now I prefer to change the keyword in the function itself to `y_grid`, i.e. add y_grid and deprecate `ygrid`.
Or for now, at least we should make a renaming y_grid to ygrid


Resolving patch:
diff --git a/statsmodels/stats/rates.py b/statsmodels/stats/rates.py
--- a/statsmodels/stats/rates.py
+++ b/statsmodels/stats/rates.py
@@ -7,6 +7,8 @@
 
 
 import numpy as np
+import warnings
+
 from scipy import stats
 
 from statsmodels.stats.base import HolderTuple
@@ -57,7 +59,7 @@ def test_poisson_2indep(count1, exposure1, count2, exposure2, ratio_null=1,
         - 'smaller' :  H1: ratio of rates is smaller than ratio_null
     etest_kwds: dictionary
         Additional parameters to be passed to the etest_poisson_2indep
-        function, namely ygrid.
+        function, namely y_grid.
 
     Returns
     -------
@@ -150,7 +152,8 @@ def test_poisson_2indep(count1, exposure1, count2, exposure2, ratio_null=1,
 
 
 def etest_poisson_2indep(count1, exposure1, count2, exposure2, ratio_null=1,
-                         method='score', alternative='2-sided', ygrid=None):
+                         method='score', alternative='2-sided', ygrid=None,
+                         y_grid=None):
     """E-test for ratio of two sample Poisson rates
 
     If the two Poisson rates are g1 and g2, then the Null hypothesis is
@@ -184,11 +187,15 @@ def etest_poisson_2indep(count1, exposure1, count2, exposure2, ratio_null=1,
            'larger' :   H1: ratio of rates is larger than ratio_null
            'smaller' :  H1: ratio of rates is smaller than ratio_null
 
-    ygrid : None or 1-D ndarray
+    y_grid : None or 1-D ndarray
         Grid values for counts of the Poisson distribution used for computing
         the pvalue. By default truncation is based on an upper tail Poisson
         quantiles.
 
+    ygrid : None or 1-D ndarray
+        Same as y_grid. Deprecated. If both y_grid and ygrid are provided,
+        ygrid will be ignored.
+
     Returns
     -------
     stat_sample : float
@@ -237,15 +244,23 @@ def stat_func(x1, x2):
 
     stat_sample = stat_func(y1, y2)
 
+    if ygrid is not None:
+        warnings.warn("ygrid is deprecated, use y_grid", DeprecationWarning)
+    y_grid = y_grid if y_grid is not None else ygrid
+
     # The following uses a fixed truncation for evaluating the probabilities
     # It will currently only work for small counts, so that sf at truncation
     # point is small
     # We can make it depend on the amount of truncated sf.
     # Some numerical optimization or checks for large means need to be added.
-    if ygrid is None:
+    if y_grid is None:
         threshold = stats.poisson.isf(1e-13, max(mean1, mean2))
         threshold = max(threshold, 100)   # keep at least 100
         y_grid = np.arange(threshold + 1)
+    else:
+        y_grid = np.asarray(y_grid)
+        if y_grid.ndim != 1:
+            raise ValueError("y_grid needs to be None or 1-dimensional array")
     pdf1 = stats.poisson.pmf(y_grid, mean1)
     pdf2 = stats.poisson.pmf(y_grid, mean2)
 


Test patch:
diff --git a/statsmodels/stats/tests/test_rates_poisson.py b/statsmodels/stats/tests/test_rates_poisson.py
--- a/statsmodels/stats/tests/test_rates_poisson.py
+++ b/statsmodels/stats/tests/test_rates_poisson.py
@@ -1,6 +1,8 @@
 
 
 import pytest
+import warnings
+from numpy import arange
 from numpy.testing import assert_allclose, assert_equal
 
 # we cannot import test_poisson_2indep directly, pytest treats that as test
@@ -245,3 +247,31 @@ def test_alternative(case):
     _, pv = smr.test_poisson_2indep(count1, n1, count2, n2, method=meth,
                                     ratio_null=1.2, alternative=alt)
     assert_allclose(pv, cases_alt[case], rtol=1e-13)
+
+
+def test_y_grid_regression():
+    y_grid = arange(1000)
+
+    _, pv = etest_poisson_2indep(60, 51477.5, 30, 54308.7, y_grid=y_grid)
+    assert_allclose(pv, 0.000567261758250953, atol=1e-15)
+
+    _, pv = etest_poisson_2indep(41, 28010, 15, 19017, y_grid=y_grid)
+    assert_allclose(pv, 0.03782053187021494, atol=1e-15)
+
+    _, pv = etest_poisson_2indep(1, 1, 1, 1, y_grid=[1])
+    assert_allclose(pv, 0.1353352832366127, atol=1e-15)
+
+
+def test_invalid_y_grid():
+    # check ygrid deprecation
+    warnings.simplefilter("always")
+    with warnings.catch_warnings(record=True) as w:
+        etest_poisson_2indep(1, 1, 1, 1, ygrid=[1])
+    assert len(w) == 1
+    assert issubclass(w[0].category, DeprecationWarning)
+    assert "ygrid" in str(w[0].message)
+
+    # check y_grid validity
+    with pytest.raises(ValueError) as e:
+        etest_poisson_2indep(1, 1, 1, 1, y_grid=1)
+    assert "y_grid" in str(e.value)