Repository: statsmodels/statsmodels

Issue:
TST scipy compat: error with pre-release testing
There are errors with unit tests in prerelease versions

looks like related to using `_lazywhere`. 
We have our own version in statsmodels.compat.scipy . IIRC it was added as a copy of the scipy version.
scipy.distributions does not broadcast args anymore in dev version

https://dev.azure.com/statsmodels/statsmodels-testing/_build/results?buildId=3199&view=logs&j=d3aeec9b-5735-51da-6bfa-db4d86dad819&t=4a8eef27-4d92-5b93-41e2-3d1348afd29c

```
______________ TestZeroInflatedPoisson_predict.test_predict_prob _______________
[gw0] linux -- Python 3.9.2 /opt/hostedtoolcache/Python/3.9.2/x64/bin/python

self = <statsmodels.discrete.tests.test_count_model.TestZeroInflatedPoisson_predict object at 0x7f738ab1f340>

    def test_predict_prob(self):
        res = self.res
    
>       pr = res.predict(which='prob')

statsmodels/discrete/tests/test_count_model.py:266: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
statsmodels/base/model.py:1103: in predict
    predict_results = self.model.predict(self.params, exog, *args,
statsmodels/discrete/count_model.py:451: in predict
    return self._predict_prob(params, exog, exog_infl, exposure, offset)
statsmodels/discrete/count_model.py:535: in _predict_prob
    result = self.distribution.pmf(counts, mu, w)
/opt/hostedtoolcache/Python/3.9.2/x64/lib/python3.9/site-packages/scipy/stats/_distn_infrastructure.py:3225: in pmf
    place(output, cond, np.clip(self._pmf(*goodargs), 0, 1))
statsmodels/distributions/discrete.py:40: in _pmf
    return np.exp(self._logpmf(x, mu, w))
statsmodels/distributions/discrete.py:34: in _logpmf
    return _lazywhere(x != 0, (x, mu, w),
statsmodels/compat/scipy.py:97: in _lazywhere
    np.place(out, cond, f(*temp))
<__array_function__ internals>:5: in place
    ???
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

arr = array([[-1.73119153, -1.73119153, -1.73119153, ..., -1.73119153,
        -1.73119153, -1.73119153],
       [-1.7311915...895, -1.28321895],
       [-1.28321895, -1.28321895, -1.28321895, ..., -1.28321895,
        -1.28321895, -1.28321895]])
mask = array([[False,  True,  True,  True,  True,  True,  True,  True,  True,
         True]])
vals = array([-1.37428414, -1.36072044, -1.75262184, -2.43220532, -3.33493235,
       -4.41998093, -5.6591802 , -7.03191085, -8.52242455])

    @array_function_dispatch(_place_dispatcher)
    def place(arr, mask, vals):
        """
        Change elements of an array based on conditional and input values.
    
        Similar to ``np.copyto(arr, vals, where=mask)``, the difference is that
        `place` uses the first N elements of `vals`, where N is the number of
        True values in `mask`, while `copyto` uses the elements where `mask`
        is True.
    
        Note that `extract` does the exact opposite of `place`.
    
        Parameters
        ----------
        arr : ndarray
            Array to put data into.
        mask : array_like
            Boolean mask array. Must have the same size as `a`.
        vals : 1-D sequence
            Values to put into `a`. Only the first N elements are used, where
            N is the number of True values in `mask`. If `vals` is smaller
            than N, it will be repeated, and if elements of `a` are to be masked,
            this sequence must be non-empty.
    
        See Also
        --------
        copyto, put, take, extract
    
        Examples
        --------
        >>> arr = np.arange(6).reshape(2, 3)
        >>> np.place(arr, arr>2, [44, 55])
        >>> arr
        array([[ 0,  1,  2],
               [44, 55, 44]])
    
        """
        if not isinstance(arr, np.ndarray):
            raise TypeError("argument 1 must be numpy.ndarray, "
                            "not {name}".format(name=type(arr).__name__))
    
>       return _insert(arr, mask, vals)
E       ValueError: place: mask and data must be the same size

/opt/hostedtoolcache/Python/3.9.2/x64/lib/python3.9/site-packages/numpy/lib/function_base.py:1748: ValueError

```


Discussion:
The relevant code is in statsmodels.distributions.discrete

eg. for Poisson, but others follow the same pattern
lazywhere in _logpmf

```
class zipoisson_gen(rv_discrete):
    '''Zero Inflated Poisson distribution
    '''
    def _argcheck(self, mu, w):
        return (mu > 0) & (w >= 0) & (w<=1)

    def _logpmf(self, x, mu, w):
        return _lazywhere(x != 0, (x, mu, w),
                          (lambda x, mu, w: np.log(1. - w) + x * np.log(mu) -
                          gammaln(x + 1.) - mu),
                          np.log(w + (1. - w) * np.exp(-mu)))

    def _pmf(self, x, mu, w):
        return np.exp(self._logpmf(x, mu, w))
```
The change in `argsreduce` depends on also changing `_lazywhere` to broadcast `cond` to the same shape as the other arrays tp avoid exactly this problem.
That's a possible change for the future in statsmodels.

The problem is existing versions will not be compatible and stop working when or if users upgrade scipy.
Similar problems might show up in other packages that subclass scipy distributions.
Understood. I was working on an opt in version of the changes, but stopped when the existing PR was committed.

For what its worth, the big change is in `argsreduce`. I think the change to `_lazywhere` is safe and also fixes scipy/scipy/#13602.

Resolving patch:
diff --git a/statsmodels/compat/scipy.py b/statsmodels/compat/scipy.py
--- a/statsmodels/compat/scipy.py
+++ b/statsmodels/compat/scipy.py
@@ -63,37 +63,3 @@ def _valarray(shape, value=np.nan, typecode=None):
     if not isinstance(out, np.ndarray):
         out = np.asarray(out)
     return out
-
-
-def _lazywhere(cond, arrays, f, fillvalue=None, f2=None):
-    """
-    np.where(cond, x, fillvalue) always evaluates x even where cond is False.
-    This one only evaluates f(arr1[cond], arr2[cond], ...).
-    For example,
-    >>> a, b = np.array([1, 2, 3, 4]), np.array([5, 6, 7, 8])
-    >>> def f(a, b):
-        return a*b
-    >>> _lazywhere(a > 2, (a, b), f, np.nan)
-    array([ nan,  nan,  21.,  32.])
-    Notice it assumes that all `arrays` are of the same shape, or can be
-    broadcasted together.
-    """
-    if fillvalue is None:
-        if f2 is None:
-            raise ValueError("One of (fillvalue, f2) must be given.")
-        else:
-            fillvalue = np.nan
-    else:
-        if f2 is not None:
-            raise ValueError("Only one of (fillvalue, f2) can be given.")
-
-    arrays = np.broadcast_arrays(*arrays)
-    temp = tuple(np.extract(cond, arr) for arr in arrays)
-    tcode = np.mintypecode([a.dtype.char for a in arrays])
-    out = _valarray(np.shape(arrays[0]), value=fillvalue, typecode=tcode)
-    np.place(out, cond, f(*temp))
-    if f2 is not None:
-        temp = tuple(np.extract(~cond, arr) for arr in arrays)
-        np.place(out, ~cond, f2(*temp))
-
-    return out
diff --git a/statsmodels/distributions/discrete.py b/statsmodels/distributions/discrete.py
--- a/statsmodels/distributions/discrete.py
+++ b/statsmodels/distributions/discrete.py
@@ -1,7 +1,7 @@
 import numpy as np
 from scipy.stats import rv_discrete, nbinom, poisson
 from scipy.special import gammaln
-from statsmodels.compat.scipy import _lazywhere
+from scipy._lib._util import _lazywhere
 
 
 class genpoisson_p_gen(rv_discrete):
diff --git a/statsmodels/genmod/_tweedie_compound_poisson.py b/statsmodels/genmod/_tweedie_compound_poisson.py
--- a/statsmodels/genmod/_tweedie_compound_poisson.py
+++ b/statsmodels/genmod/_tweedie_compound_poisson.py
@@ -17,7 +17,7 @@
     insurance claims data: dispersion modelling. ASTIN Bulletin 32: 143–157
 """
 import numpy as np
-from scipy._lib._util import _lazywhere as lazywhere
+from scipy._lib._util import _lazywhere
 from scipy.special import gammaln
 
 
@@ -76,10 +76,10 @@ def density_otherwise(y, mu, p, phi):
 
 
 def series_density(y, mu, p, phi):
-    density = lazywhere(np.array(y) > 0,
-                        (y, mu, p, phi),
-                        f=density_otherwise,
-                        f2=density_at_zero)
+    density = _lazywhere(np.array(y) > 0,
+                         (y, mu, p, phi),
+                         f=density_otherwise,
+                         f2=density_at_zero)
     return density
 
 


Test patch: