Repository: scipy/scipy

Issue:
BUG: multivariate_normal returns a pdf for values outside its support when covariance is singular
When a multivariate normal distribution has a singular covariance matrix, its support (i.e. the set of possible values the random variable can take) is restricted to a manifold. In this case, the density with respect to the usual Lebesgue measure is not defined. There IS a density with respect to the Hausdorff measure on the support, and this is exactly what stats.multivariate_normal returns. The problem is that it returns a pdf for any input. Specifically, it returns the density of the projection of the input onto the support. I don't think this behaviour is correct. To demonstrate:

``` python
>>> import numpy as np
>>> from scipy.stats import multivariate_normal
>>> mean = np.zeros(2)
>>> cov = np.ones((2,2))
>>> rv = multivariate_normal(mean=mean, cov=cov, allow_singular=True)
>>> x = np.array([[0.0, 0.0], [1.0, -1.0], [2.0, -2.0]])
>>> x
array([[ 0.,  0.],
       [ 1., -1.],
       [ 2., -2.]])
>>> rv.pdf(x)
array([ 0.28209479,  0.28209479,  0.28209479])
```

For this distribution, the random variable must lie on the line `x[0]-x[1]=0`. `[0,0]` is within the support, so that's all well and good. The other two points are not. Technically, the density is simply undefined at these points. For consistency with other scipy.stats distributions, the output should probably be `0.0`.



Discussion:

Resolving patch:
diff --git a/scipy/stats/_multivariate.py b/scipy/stats/_multivariate.py
--- a/scipy/stats/_multivariate.py
+++ b/scipy/stats/_multivariate.py
@@ -168,14 +168,26 @@ def __init__(self, M, cond=None, rcond=None, lower=True,
         s_pinv = _pinv_1d(s, eps)
         U = np.multiply(u, np.sqrt(s_pinv))
 
+        # Save the eigenvector basis, and tolerance for testing support
+        self.eps = 1e3*eps
+        self.V = u[:, s <= eps]
+
         # Initialize the eagerly precomputed attributes.
         self.rank = len(d)
         self.U = U
         self.log_pdet = np.sum(np.log(d))
 
-        # Initialize an attribute to be lazily computed.
+        # Initialize attributes to be lazily computed.
         self._pinv = None
 
+    def _support_mask(self, x):
+        """
+        Check whether x lies in the support of the distribution.
+        """
+        residual = np.linalg.norm(x @ self.V, axis=-1)
+        in_support = residual < self.eps
+        return in_support
+
     @property
     def pinv(self):
         if self._pinv is None:
@@ -312,10 +324,16 @@ class multivariate_normal_gen(multi_rv_generic):
                \exp\left( -\frac{1}{2} (x - \mu)^T \Sigma^{-1} (x - \mu) \right),
 
     where :math:`\mu` is the mean, :math:`\Sigma` the covariance matrix,
-    and :math:`k` is the dimension of the space where :math:`x` takes values.
+    :math:`k` the rank of :math:`\Sigma`. In case of singular :math:`\Sigma`,
+    SciPy extends this definition according to [1]_.
 
     .. versionadded:: 0.14.0
 
+    References
+    ----------
+    .. [1] Multivariate Normal Distribution - Degenerate Case, Wikipedia,
+           https://en.wikipedia.org/wiki/Multivariate_normal_distribution#Degenerate_case
+
     Examples
     --------
     >>> import matplotlib.pyplot as plt
@@ -494,6 +512,9 @@ def logpdf(self, x, mean=None, cov=1, allow_singular=False):
         x = self._process_quantiles(x, dim)
         psd = _PSD(cov, allow_singular=allow_singular)
         out = self._logpdf(x, mean, psd.U, psd.log_pdet, psd.rank)
+        if allow_singular and (psd.rank < dim):
+            out_of_bounds = ~psd._support_mask(x-mean)
+            out[out_of_bounds] = -np.inf
         return _squeeze_output(out)
 
     def pdf(self, x, mean=None, cov=1, allow_singular=False):
@@ -519,6 +540,9 @@ def pdf(self, x, mean=None, cov=1, allow_singular=False):
         x = self._process_quantiles(x, dim)
         psd = _PSD(cov, allow_singular=allow_singular)
         out = np.exp(self._logpdf(x, mean, psd.U, psd.log_pdet, psd.rank))
+        if allow_singular and (psd.rank < dim):
+            out_of_bounds = ~psd._support_mask(x-mean)
+            out[out_of_bounds] = 0.0
         return _squeeze_output(out)
 
     def _cdf(self, x, mean, cov, maxpts, abseps, releps):
@@ -728,6 +752,7 @@ def __init__(self, mean=None, cov=1, allow_singular=False, seed=None,
         array([[1.]])
 
         """
+        self.allow_singular = allow_singular
         self._dist = multivariate_normal_gen(seed)
         self.dim, self.mean, self.cov = self._dist._process_parameters(
                                                             None, mean, cov)
@@ -742,6 +767,9 @@ def logpdf(self, x):
         x = self._dist._process_quantiles(x, self.dim)
         out = self._dist._logpdf(x, self.mean, self.cov_info.U,
                                  self.cov_info.log_pdet, self.cov_info.rank)
+        if self.allow_singular and (self.cov_info.rank < self.dim):
+            out_of_bounds = ~self.cov_info._support_mask(x-self.mean)
+            out[out_of_bounds] = -np.inf
         return _squeeze_output(out)
 
     def pdf(self, x):


Test patch:
diff --git a/scipy/stats/tests/test_multivariate.py b/scipy/stats/tests/test_multivariate.py
--- a/scipy/stats/tests/test_multivariate.py
+++ b/scipy/stats/tests/test_multivariate.py
@@ -35,6 +35,12 @@
 from unittest.mock import patch
 
 
+def _sample_orthonormal_matrix(n):
+    M = np.random.randn(n, n)
+    u, s, v = scipy.linalg.svd(M)
+    return u
+
+
 class TestMultivariateNormal:
     def test_input_shape(self):
         mu = np.arange(3)
@@ -130,22 +136,21 @@ def test_rank(self):
 
     def test_degenerate_distributions(self):
 
-        def _sample_orthonormal_matrix(n):
-            M = np.random.randn(n, n)
-            u, s, v = scipy.linalg.svd(M)
-            return u
-
         for n in range(1, 5):
-            x = np.random.randn(n)
-            for k in range(1, n + 1):
+            z = np.random.randn(n)
+            for k in range(1, n):
                 # Sample a small covariance matrix.
                 s = np.random.randn(k, k)
                 cov_kk = np.dot(s, s.T)
 
-                # Embed the small covariance matrix into a larger low rank matrix.
+                # Embed the small covariance matrix into a larger singular one.
                 cov_nn = np.zeros((n, n))
                 cov_nn[:k, :k] = cov_kk
 
+                # Embed part of the vector in the same way
+                x = np.zeros(n)
+                x[:k] = z[:k]
+
                 # Define a rotation of the larger low rank matrix.
                 u = _sample_orthonormal_matrix(n)
                 cov_rr = np.dot(u, np.dot(cov_nn, u.T))
@@ -172,6 +177,37 @@ def _sample_orthonormal_matrix(n):
                 assert_allclose(logpdf_kk, logpdf_nn)
                 assert_allclose(logpdf_kk, logpdf_rr)
 
+                # Add an orthogonal component and find the density
+                y_orth = y + u[:, -1]
+                pdf_rr_orth = distn_rr.pdf(y_orth)
+                logpdf_rr_orth = distn_rr.logpdf(y_orth)
+
+                # Ensure that this has zero probability
+                assert_equal(pdf_rr_orth, 0.0)
+                assert_equal(logpdf_rr_orth, -np.inf)
+
+    def test_degenerate_array(self):
+        # Test that we can generate arrays of random variate from a degenerate
+        # multivariate normal, and that the pdf for these samples is non-zero
+        # (i.e. samples from the distribution lie on the subspace)
+        k = 10
+        for n in range(2, 6):
+            for r in range(1, n):
+                mn = np.zeros(n)
+                u = _sample_orthonormal_matrix(n)[:, :r]
+                vr = np.dot(u, u.T)
+                X = multivariate_normal.rvs(mean=mn, cov=vr, size=k)
+
+                pdf = multivariate_normal.pdf(X, mean=mn, cov=vr,
+                                              allow_singular=True)
+                assert_equal(pdf.size, k)
+                assert np.all(pdf > 0.0)
+
+                logpdf = multivariate_normal.logpdf(X, mean=mn, cov=vr,
+                                                    allow_singular=True)
+                assert_equal(logpdf.size, k)
+                assert np.all(logpdf > -np.inf)
+
     def test_large_pseudo_determinant(self):
         # Check that large pseudo-determinants are handled appropriately.