Repository: scipy/scipy

Issue:
BUG: beta.pdf is broken on main (1.11.0.dev0)
### Describe your issue.

beta.pdf has changes from 1.10 to the 1.11.0.dev0 build and now returns incorrect results.




### Reproducing Code Example

```python
import numpy as np
from scipy import stats

xx = np.array([0.        , 0.11111111, 0.22222222, 0.33333333, 0.44444444, 0.55555556, 0.66666667, 0.77777778, 0.88888889, 1.        ])
k = np.array([0., 1., 2., 3., 4.])
n=4.0

pdf = stats.beta.pdf(xx[..., None], k + 1, n - k + 1)


np.testing.assert_allclose(pdf[-1,-1],5.0)

```
Passes on 1.10.0, fails on 1.11.0.dev0.



### Error message

```shell
AssertionError: 
Not equal to tolerance rtol=1e-07, atol=0
Mismatched elements: 1 / 1 (100%)
Max absolute difference: 5.
Max relative difference: 1.
 x: array(0.)
 y: array(5.)
```


### SciPy/NumPy/Python version and system information

```shell
1.11.0.dev0+1817.98c51a6 1.23.5 sys.version_info(major=3, minor=11, micro=2, releaselevel='final', serial=0)
{
  "Compilers": {
    "c": {
      "name": "gcc",
      "linker": "ld.bfd",
      "version": "10.3.0",
      "commands": "cc"
    },
    "cython": {
      "name": "cython",
      "linker": "cython",
      "version": "0.29.34",
      "commands": "cython"
    },
    "c++": {
      "name": "gcc",
      "linker": "ld.bfd",
      "version": "10.3.0",
      "commands": "c++"
    },
    "fortran": {
      "name": "gcc",
      "linker": "ld.bfd",
      "version": "10.3.0",
      "commands": "gfortran"
    },
    "pythran": {
      "version": "0.12.1",
      "include directory": "C:\\Users\\runneradmin\\AppData\\Local\\Temp\\pip-build-env-024pp_ot\\overlay\\Lib\\site-packages/pythran"
    }
  },
  "Machine Information": {
    "host": {
      "cpu": "x86_64",
      "family": "x86_64",
      "endian": "little",
      "system": "windows"
    },
    "build": {
      "cpu": "x86_64",
      "family": "x86_64",
      "endian": "little",
      "system": "windows"
    },
    "cross-compiled": false
  },
  "Build Dependencies": {
    "blas": {
      "name": "openblas",
      "found": true,
      "version": "0.3.21.dev",
      "detection method": "pkgconfig",
      "include directory": "/c/opt/64/include",
      "lib directory": "/c/opt/64/lib",
      "openblas configuration": "USE_64BITINT= DYNAMIC_ARCH=1 DYNAMIC_OLDER= NO_CBLAS= NO_LAPACK= NO_LAPACKE= NO_AFFINITY=1 USE_OPENMP= SKYLAKEX MAX_THREADS=2",
      "pc file directory": "c:/opt/64/lib/pkgconfig"
    },
    "lapack": {
      "name": "openblas",
      "found": true,
      "version": "0.3.21.dev",
      "detection method": "pkgconfig",
      "include directory": "/c/opt/64/include",
      "lib directory": "/c/opt/64/lib",
      "openblas configuration": "USE_64BITINT= DYNAMIC_ARCH=1 DYNAMIC_OLDER= NO_CBLAS= NO_LAPACK= NO_LAPACKE= NO_AFFINITY=1 USE_OPENMP= SKYLAKEX MAX_THREADS=2",
      "pc file directory": "c:/opt/64/lib/pkgconfig"
    }
  },
  "Python Information": {
    "path": "C:\\Users\\runneradmin\\AppData\\Local\\Temp\\cibw-run-cgdr6yw7\\cp311-win_amd64\\build\\venv\\Scripts\\python.exe",
    "version": "3.11"
  }
}
```
```



Discussion:
Creating a failure on statsmodels pre-release testing.

Thanks @bashtage I confirm the issue. A smaller reproducer is

```python
stats.beta.pdf(1e-30, a=1, b=5)
# 5.000000000000035
stats.beta.pdf(0, a=1, b=5)
# 0.0
```
I can get to the actually error playing with the float

```python
stats.beta.pdf(1e-310, a=1, b=5)
Traceback (most recent call last):
  File ".../envs/scipy-dev-311/lib/python3.11/site-packages/IPython/core/interactiveshell.py", line 3505, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "<ipython-input-50-530e9aaceec0>", line 1, in <module>
    stats.beta.pdf(1e-310, a=1, b=5)
  File ".../scipy/build-install/lib/python3.11/site-packages/scipy/stats/_distn_infrastructure.py", line 1992, in pdf
    place(output, cond, self._pdf(*goodargs) / scale)
                        ^^^^^^^^^^^^^^^^^^^^
  File ".../scipy/build-install/lib/python3.11/site-packages/scipy/stats/_continuous_distns.py", line 681, in _pdf
    return _boost._beta_pdf(x, a, b)
           ^^^^^^^^^^^^^^^^^^^^^^^^^
OverflowError: Error in function ibeta_derivative<d>(%1%,%1%,%1%): Overflow Error
```

Seems like it's a boost thing. cc @mckib2 
Ping @jzmaddock @mborland 
Interesting; I will look at it. Does Scipy track the current Boost.Math develop, or is pinned to a particular commit?
Thanks @mborland!

It's tracking `7203fa2def6347b0d5f8fe1e8522d5b0a618db9d` but I could reproduce on `109a814e89f77ff8a3fc8f0391f6b35a12640669` (develop). If I did this correctly, I updated and rebuilt from scratch 😅
There are two issues: the return value at the boundary when the shape parameter associated with that boundary is 1, and the overflow when `x` is subnormal (e.g. 1e-310).  @mborland has a fix in progress at https://github.com/boostorg/math/pull/977.

In SciPy 1.10.1, `beta.pdf(0, 1, 5)` returns 5.  That is the correct limit as $x \rightarrow 0^+$.  Currently, `beta.pdf(0, 1, 5)` returns 0, and that appears to be what the beta distribution in boost/math has always done (I haven't checked the history thoroughly).  The justification there is that Wolfram Alpha returns 0.  But the "correct" value of the PDF at the boundaries is more often a matter of convention than mathematical rigor (@rkern wrote a nice summary of this, but I don't recall where).  We can restore the old convention in SciPy by adding an additional check for this special case here:

https://github.com/scipy/scipy/blob/b07ed536bdd1bee735e0ded1dbe5bf32ac209604/scipy/stats/_boost/include/func_defs.hpp#L71-L88

E.g. this should do it:

```C++
template<template <typename, typename> class Dst, class RealType, class...Args>
RealType
boost_pdf_beta(const RealType x, const RealType a, const RealType b)
{
    if (std::isfinite(x)) {
        if ((x >= 1) && (b < 1)) {
            // x>1 should really be 0, but rv_continuous will do that for us
            return INFINITY;
        }
        else if ((x <= 0) && (a < 1)) {
            return INFINITY;
        }
        else if ((x == 0) && (a == 1)) {
            return b;
        }
        else if ((x == 1) && (b == 1)) {
            return a;
        }
        return boost::math::pdf(
	    boost::math::beta_distribution<RealType, Policy>(a, b), x);
    }
    return NAN;
}
```
For Wolfram alpha, not sure how you checked. I personally just looked at the graph here https://www.wolframalpha.com/input?i=Beta%281%2C+5%29&assumption=%7B%22C%22%2C+%22Beta%22%7D+-%3E+%7B%22ProbabilityDistribution%22%7D
You can use [`PDF[BetaDistribution[1, 5], 0]`](https://www.wolframalpha.com/input?i=PDF%5BBetaDistribution%5B1%2C+5%5D%2C+0%5D).
What do you get with scipy main for the upper bound?

```
beta.pdf(0, 1, 5), beta.pdf(1, 5, 1)
(5.0, 5.0)
scipy.__version__
'1.7.3'
```

The application in statsmodels is for bernstein polynomials.
With this old behavior of beta.pdf at the boundaries, a theoretically implied round trip test for bernstein polynomials also works at the 0, 1 boundary values. This test fails now at the upper boundary with scipy main.

And it looks like those are the correct values by continuity.

I cannot get the error message, but otherwise same story

```python
stats.beta.pdf(1-1e-16, a=5, b=1)
# 4.999999999999993
stats.beta.pdf(1-1e-17, a=5, b=1)
# 0.0
```
> What do you get with scipy main for the upper bound?

In main, `beta.pdf(0, 1, 5), beta.pdf(1, 5, 1)` will give `(0.0, 0.0)`.  This is exactly the issue that @bashtage raised and @tupui distilled to a minimal reproducer.  With my suggestion above, the old behavior would be restored, and the result would be `(5.0, 5.0)`.
aside: Wolfram alpha defines the Beta distribution on the open interval.
So boundary points are outside of the support and so pdf is zero.

If both parameters are >= 1, then there is no problem with infinite pdf at the boundary.
Thanks for the discussion everyone, much appreciated.

I think we have a few issues here:

1) if alpha < 1 and x is zero, or beta < 1 and x is 1, then we are just plain wrong, and should be returning infinity (overflow_error in our case).  @mborland the offending code is here: https://github.com/boostorg/math/blob/109a814e89f77ff8a3fc8f0391f6b35a12640669/include/boost/math/distributions/beta.hpp#L395
2) If alpha = 1 and x is 0, or beta = 1 and x = 1, then we end up with 0^0 in the definition of the PDF which is undefined.  So we're not *wrong* as such, but I agree that returning the limit (`1/beta(alpha, beta)`) would be preferable.

Hopefully I have that right ;)

@mborland are you OK to take care of this?
The linked PR for boost.math has been merged. Let me know if that solves your issues.
Thank you @mborland I am having a look and will make a PR here to update the submodule if it's fixed 😃 
I confirm it's fixed 🚀 I am making a PR to update the submodule now. Thanks again everyone!

Resolving patch:
diff --git a/scipy/stats/_continuous_distns.py b/scipy/stats/_continuous_distns.py
--- a/scipy/stats/_continuous_distns.py
+++ b/scipy/stats/_continuous_distns.py
@@ -678,7 +678,8 @@ def _pdf(self, x, a, b):
         #                     gamma(a+b) * x**(a-1) * (1-x)**(b-1)
         # beta.pdf(x, a, b) = ------------------------------------
         #                              gamma(a)*gamma(b)
-        return _boost._beta_pdf(x, a, b)
+        with np.errstate(over='ignore'):
+            return _boost._beta_pdf(x, a, b)
 
     def _logpdf(self, x, a, b):
         lPx = sc.xlog1py(b - 1.0, -x) + sc.xlogy(a - 1.0, x)
@@ -697,8 +698,6 @@ def _isf(self, x, a, b):
 
     def _ppf(self, q, a, b):
         with np.errstate(over='ignore'):  # see gh-17432
-            message = "overflow encountered in _beta_ppf"
-            warnings.filterwarnings('ignore', message=message)
             return _boost._beta_ppf(q, a, b)
 
     def _stats(self, a, b):


Test patch:
diff --git a/scipy/stats/tests/test_boost_ufuncs.py b/scipy/stats/tests/test_boost_ufuncs.py
--- a/scipy/stats/tests/test_boost_ufuncs.py
+++ b/scipy/stats/tests/test_boost_ufuncs.py
@@ -20,6 +20,8 @@
     (_boost._beta_cdf, (0.5, 2, 3), 0.6875),
     (_boost._beta_ppf, (0.6875, 2, 3), 0.5),
     (_boost._beta_pdf, (0.5, 2, 3), 1.5),
+    (_boost._beta_pdf, (0, 1, 5), 5.0),
+    (_boost._beta_pdf, (1, 5, 1), 5.0),
     (_boost._beta_sf, (0.5, 2, 1), 0.75),
     (_boost._beta_isf, (0.75, 2, 1), 0.5),
     (_boost._binom_cdf, (1, 3, 0.5), 0.5),
diff --git a/scipy/stats/tests/test_distributions.py b/scipy/stats/tests/test_distributions.py
--- a/scipy/stats/tests/test_distributions.py
+++ b/scipy/stats/tests/test_distributions.py
@@ -4040,6 +4040,18 @@ def test_endpoints(self):
         a, b = 0.2, 3
         assert_equal(stats.beta.pdf(0, a, b), np.inf)
 
+        # Confirm that boost's beta distribution returns 5 at x=0
+        # when a=1, b=5
+        a, b = 1, 5
+        assert_equal(stats.beta.pdf(0, a, b), 5)
+        assert_equal(stats.beta.pdf(1e-310, a, b), 5)
+
+        # Confirm that boost's beta distribution returns 5 at x=1
+        # when a=5, b=1
+        a, b = 5, 1
+        assert_equal(stats.beta.pdf(1, a, b), 5)
+        assert_equal(stats.beta.pdf(1-1e-310, a, b), 5)
+
     @pytest.mark.xfail(IS_PYPY, reason="Does not convert boost warning")
     def test_boost_eval_issue_14606(self):
         q, a, b = 0.995, 1.0e11, 1.0e13