Repository: scipy/scipy

Issue:
Make it possible to pass a rank cut-off value relatively to the largest singular value in pinv2
It would be great to have distinct parameters to control the cut-off threshold to ignore low singular values when computing the pseudo-inverse, either in terms of absolute values or values relative to the maximum singular value computed by the internal SVD of pinv2.

As discussed in https://github.com/scipy/scipy/pull/10067#issuecomment-799499529.

I think this was the pupose of having 2 distinct parameters  `rcond` and `cond` but at the moment, and even before the change in  #10067, passing `rcond` effectively results in the same behavior as passing `cond`.

Furthermore the docstring is not explicit enough and the names themselves are too vague.

Note that there are probably other places in scipy with those `cond` and `rcond` parameters. I did not conduct an audit to check if they have similar problems or not.

/cc @ilayn 


Discussion:
This would be required to implement a clean fix to a regression we observed in scikit-learn: 

https://github.com/scikit-learn/scikit-learn/pull/19646

At the moment we maintain a local private backport of pinv2 as a workaround.

Resolving patch:
diff --git a/scipy/linalg/basic.py b/scipy/linalg/basic.py
--- a/scipy/linalg/basic.py
+++ b/scipy/linalg/basic.py
@@ -1233,46 +1233,67 @@ def lstsq(a, b, cond=None, overwrite_a=False, overwrite_b=False,
 lstsq.default_lapack_driver = 'gelsd'
 
 
-def pinv(a, cond=None, rcond=None, return_rank=False, check_finite=True):
+def pinv(a, atol=None, rtol=None, return_rank=False, check_finite=True,
+          cond=None, rcond=None):
     """
     Compute the (Moore-Penrose) pseudo-inverse of a matrix.
 
-    Calculate a generalized inverse of a matrix using a least-squares
-    solver.
+    Calculate a generalized inverse of a matrix using its
+    singular-value decomposition ``U @ S @ V`` in the economy mode and picking
+    up only the columns/rows that are associated with significant singular
+    values.
+
+    If ``s`` is the maximum singular value of ``a``, then the
+    significance cut-off value is determined by ``atol + rtol * s``. Any
+    singular value below this value is assumed insignificant.
 
     Parameters
     ----------
     a : (M, N) array_like
         Matrix to be pseudo-inverted.
-    cond, rcond : float, optional
-        Cutoff factor for 'small' singular values. In `lstsq`,
-        singular values less than ``cond*largest_singular_value`` will be
-        considered as zero. If both are omitted, the default value
-        ``max(M, N) * eps`` is passed to `lstsq` where ``eps`` is the
-        corresponding machine precision value of the datatype of ``a``.
+    atol: float, optional
+        Absolute threshold term, default value is 0.
 
-        .. versionchanged:: 1.3.0
-            Previously the default cutoff value was just `eps` without the
-            factor ``max(M, N)``.
+        .. versionadded:: 1.7.0
+
+    rtol: float, optional
+        Relative threshold term, default value is ``max(M, N) * eps`` where
+        ``eps`` is the machine precision value of the datatype of ``a``.
+
+        .. versionadded:: 1.7.0
 
     return_rank : bool, optional
-        if True, return the effective rank of the matrix
+        If True, return the effective rank of the matrix.
     check_finite : bool, optional
         Whether to check that the input matrix contains only finite numbers.
         Disabling may give a performance gain, but may result in problems
         (crashes, non-termination) if the inputs do contain infinities or NaNs.
+    cond, rcond : float, optional
+        In older versions, these values were meant to be used as ``atol`` with
+        ``rtol=0``. If both were given ``rcond`` overwrote ``cond`` and hence
+        the code was not correct. Thus using these are strongly discouraged and
+        the tolerances above are recommended instead. In fact, if provided,
+        atol, rtol takes precedence over these keywords.
+
+        .. versionchanged:: 1.7.0
+            Deprecated in favor of ``rtol`` and ``atol`` parameters above and
+            will be removed in future versions of SciPy.
+
+        .. versionchanged:: 1.3.0
+            Previously the default cutoff value was just ``eps*f`` where ``f``
+            was ``1e3`` for single precision and ``1e6`` for double precision.
 
     Returns
     -------
     B : (N, M) ndarray
         The pseudo-inverse of matrix `a`.
     rank : int
-        The effective rank of the matrix. Returned if return_rank == True
+        The effective rank of the matrix. Returned if `return_rank` is True.
 
     Raises
     ------
     LinAlgError
-        If computation does not converge.
+        If SVD computation does not converge.
 
     Examples
     --------
@@ -1280,36 +1301,53 @@ def pinv(a, cond=None, rcond=None, return_rank=False, check_finite=True):
     >>> rng = np.random.default_rng()
     >>> a = rng.standard_normal((9, 6))
     >>> B = linalg.pinv(a)
-    >>> np.allclose(a, np.dot(a, np.dot(B, a)))
+    >>> np.allclose(a, a @ B @ a)
     True
-    >>> np.allclose(B, np.dot(B, np.dot(a, B)))
+    >>> np.allclose(B, B @ a @ B)
     True
 
     """
     a = _asarray_validated(a, check_finite=check_finite)
-    # If a is sufficiently tall it is cheaper to compute using the transpose
-    trans = a.shape[0] / a.shape[1] >= 1.1
-    b = np.eye(a.shape[1] if trans else a.shape[0], dtype=a.dtype)
+    u, s, vh = decomp_svd.svd(a, full_matrices=False, check_finite=False)
+    t = u.dtype.char.lower()
+    maxS = np.max(s)
 
-    if rcond is not None:
-        cond = rcond
+    if rcond or cond:
+        warn('Use of the "cond" and "rcond" keywords are deprecated and '
+             'will be removed in future versions of SciPy. Use "atol" and '
+             '"rtol" keywords instead', DeprecationWarning, stacklevel=2)
 
-    if cond is None:
-        cond = max(a.shape) * np.spacing(a.real.dtype.type(1))
+    # backwards compatible only atol and rtol are both missing
+    if (rcond or cond) and (atol is None) and (rtol is None):
+        atol = rcond or cond
+        rtol = 0.
+
+    atol = 0. if atol is None else atol
+    rtol = max(a.shape) * np.finfo(t).eps if (rtol is None) else rtol
 
-    x, resids, rank, s = lstsq(a.T if trans else a, b,
-                               cond=cond, check_finite=False)
+    if (atol < 0.) or (rtol < 0.):
+        raise ValueError("atol and rtol values must be positive.")
+
+    val = atol + maxS * rtol
+    rank = np.sum(s > val)
+
+    u = u[:, :rank]
+    u /= s[:rank]
+    B = (u @ vh[:rank]).conj().T
 
     if return_rank:
-        return (x.T if trans else x), rank
+        return B, rank
     else:
-        return x.T if trans else x
+        return B
 
 
 def pinv2(a, cond=None, rcond=None, return_rank=False, check_finite=True):
     """
     Compute the (Moore-Penrose) pseudo-inverse of a matrix.
 
+    `scipy.linalg.pinv2` is deprecated since SciPy 1.7.0, use
+    `scipy.linalg.pinv` instead for better tolerance control.
+
     Calculate a generalized inverse of a matrix using its
     singular-value decomposition and including all 'large' singular
     values.
@@ -1347,61 +1385,40 @@ def pinv2(a, cond=None, rcond=None, return_rank=False, check_finite=True):
     LinAlgError
         If SVD computation does not converge.
 
-    Examples
-    --------
-    >>> from scipy import linalg
-    >>> rng = np.random.default_rng()
-    >>> a = rng.standard_normal((9, 6))
-    >>> B = linalg.pinv2(a)
-    >>> np.allclose(a, np.dot(a, np.dot(B, a)))
-    True
-    >>> np.allclose(B, np.dot(B, np.dot(a, B)))
-    True
-
     """
-    a = _asarray_validated(a, check_finite=check_finite)
-    u, s, vh = decomp_svd.svd(a, full_matrices=False, check_finite=False)
-
+    # SciPy 1.7.0 2021-04-10
+    warn('scipy.linalg.pinv2 is deprecated since SciPy 1.7.0, use '
+         'scipy.linalg.pinv instead', DeprecationWarning, stacklevel=2)
     if rcond is not None:
         cond = rcond
-    if cond in [None, -1]:
-        t = u.dtype.char.lower()
-        cond = np.max(s) * max(a.shape) * np.finfo(t).eps
 
-    rank = np.sum(s > cond)
+    return pinv(a=a, atol=cond, rtol=None, return_rank=return_rank,
+                check_finite=check_finite)
 
-    u = u[:, :rank]
-    u /= s[:rank]
-    B = np.transpose(np.conjugate(np.dot(u, vh[:rank])))
 
-    if return_rank:
-        return B, rank
-    else:
-        return B
-
-
-def pinvh(a, cond=None, rcond=None, lower=True, return_rank=False,
-          check_finite=True):
+def pinvh(a, atol=None, rtol=None, lower=True, return_rank=False,
+          check_finite=True, cond=None, rcond=None):
     """
     Compute the (Moore-Penrose) pseudo-inverse of a Hermitian matrix.
 
-    Calculate a generalized inverse of a Hermitian or real symmetric matrix
-    using its eigenvalue decomposition and including all eigenvalues with
-    'large' absolute value.
+    Calculate a generalized inverse of a copmlex Hermitian/real symmetric
+    matrix using its eigenvalue decomposition and including all eigenvalues
+    with 'large' absolute value.
 
     Parameters
     ----------
     a : (N, N) array_like
         Real symmetric or complex hermetian matrix to be pseudo-inverted
-    cond, rcond : float or None
-        Cutoff for 'small' singular values; singular values smaller than this
-        value are considered as zero. If both are omitted, the default
-        ``max(M,N)*largest_eigenvalue*eps`` is used where ``eps`` is the
-        machine precision value of the datatype of ``a``.
+    atol: float, optional
+        Absolute threshold term, default value is 0.
 
-        .. versionchanged:: 1.3.0
-            Previously the default cutoff value was just ``eps*f`` where ``f``
-            was ``1e3`` for single precision and ``1e6`` for double precision.
+        .. versionadded:: 1.7.0
+
+    rtol: float, optional
+        Relative threshold term, default value is ``N * eps`` where
+        ``eps`` is the machine precision value of the datatype of ``a``.
+
+        .. versionadded:: 1.7.0
 
     lower : bool, optional
         Whether the pertinent array data is taken from the lower or upper
@@ -1412,6 +1429,20 @@ def pinvh(a, cond=None, rcond=None, lower=True, return_rank=False,
         Whether to check that the input matrix contains only finite numbers.
         Disabling may give a performance gain, but may result in problems
         (crashes, non-termination) if the inputs do contain infinities or NaNs.
+    cond, rcond : float, optional
+        In older versions, these values were meant to be used as ``atol`` with
+        ``rtol=0``. If both were given ``rcond`` overwrote ``cond`` and hence
+        the code was not correct. Thus using these are strongly discouraged and
+        the tolerances above are recommended instead.  In fact, if provided,
+        atol, rtol takes precedence over these keywords.
+
+        .. versionchanged:: 1.7.0
+            Deprecated in favor of ``rtol`` and ``atol`` parameters above and
+            will be removed in future versions of SciPy.
+
+        .. versionchanged:: 1.3.0
+            Previously the default cutoff value was just ``eps*f`` where ``f``
+            was ``1e3`` for single precision and ``1e6`` for double precision.
 
     Returns
     -------
@@ -1423,7 +1454,7 @@ def pinvh(a, cond=None, rcond=None, lower=True, return_rank=False,
     Raises
     ------
     LinAlgError
-        If eigenvalue does not converge
+        If eigenvalue algorithm does not converge.
 
     Examples
     --------
@@ -1432,27 +1463,40 @@ def pinvh(a, cond=None, rcond=None, lower=True, return_rank=False,
     >>> a = rng.standard_normal((9, 6))
     >>> a = np.dot(a, a.T)
     >>> B = pinvh(a)
-    >>> np.allclose(a, np.dot(a, np.dot(B, a)))
+    >>> np.allclose(a, a @ B @ a)
     True
-    >>> np.allclose(B, np.dot(B, np.dot(a, B)))
+    >>> np.allclose(B, B @ a @ B)
     True
 
     """
     a = _asarray_validated(a, check_finite=check_finite)
     s, u = decomp.eigh(a, lower=lower, check_finite=False)
+    t = u.dtype.char.lower()
+    maxS = np.max(np.abs(s))
 
-    if rcond is not None:
-        cond = rcond
-    if cond in [None, -1]:
-        t = u.dtype.char.lower()
-        cond = np.max(np.abs(s)) * max(a.shape) * np.finfo(t).eps
+    if rcond or cond:
+        warn('Use of the "cond" and "rcond" keywords are deprecated and '
+             'will be removed in future versions of SciPy. Use "atol" and '
+             '"rtol" keywords instead', DeprecationWarning, stacklevel=2)
+
+    # backwards compatible only atol and rtol are both missing
+    if (rcond or cond) and (atol is None) and (rtol is None):
+        atol = rcond or cond
+        rtol = 0.
+
+    atol = 0. if atol is None else atol
+    rtol = max(a.shape) * np.finfo(t).eps if (rtol is None) else rtol
+
+    if (atol < 0.) or (rtol < 0.):
+        raise ValueError("atol and rtol values must be positive.")
+
+    val = atol + maxS * rtol
+    above_cutoff = (abs(s) > val)
 
-    # For Hermitian matrices, singular values equal abs(eigenvalues)
-    above_cutoff = (abs(s) > cond)
     psigma_diag = 1.0 / s[above_cutoff]
     u = u[:, above_cutoff]
 
-    B = np.dot(u * psigma_diag, np.conjugate(u).T)
+    B = (u * psigma_diag) @ u.conj().T
 
     if return_rank:
         return B, len(psigma_diag)


Test patch:
diff --git a/scipy/linalg/tests/test_basic.py b/scipy/linalg/tests/test_basic.py
--- a/scipy/linalg/tests/test_basic.py
+++ b/scipy/linalg/tests/test_basic.py
@@ -15,7 +15,7 @@
 from scipy.linalg import (solve, inv, det, lstsq, pinv, pinv2, pinvh, norm,
                           solve_banded, solveh_banded, solve_triangular,
                           solve_circulant, circulant, LinAlgError, block_diag,
-                          matrix_balance, LinAlgWarning)
+                          matrix_balance, qr, LinAlgWarning)
 
 from scipy.linalg._testutils import assert_no_overwrite
 from scipy._lib._testutils import check_free_memory
@@ -1213,7 +1213,10 @@ def test_zero_size(self):
             assert_equal(s, np.empty((0,)))
 
 
+@pytest.mark.filterwarnings('ignore::DeprecationWarning')
 class TestPinv:
+    def setup_method(self):
+        np.random.seed(1234)
 
     def test_simple_real(self):
         a = array([[1, 2, 3], [4, 5, 6], [7, 8, 10]], dtype=float)
@@ -1262,16 +1265,40 @@ def test_native_list_argument(self):
         a_pinv2 = pinv2(a)
         assert_array_almost_equal(a_pinv, a_pinv2)
 
-    def test_tall_transposed(self):
-        a = random([10, 2])
-        a_pinv = pinv(a)
-        # The result will be transposed internally hence will be a C-layout
-        # instead of the typical LAPACK output with Fortran-layout
-        assert a_pinv.flags['C_CONTIGUOUS']
-
-
+    def test_atol_rtol(self):
+        n = 12
+        # get a random ortho matrix for shuffling
+        q, _ = qr(np.random.rand(n, n))
+        a_m = np.arange(35.0).reshape(7,5)
+        a = a_m.copy()
+        a[0,0] = 0.001
+        atol = 1e-5
+        rtol = 0.05
+        # svds of a_m is ~ [116.906, 4.234, tiny, tiny, tiny]
+        # svds of a is ~ [116.906, 4.234, 4.62959e-04, tiny, tiny]
+        # Just abs cutoff such that we arrive at a_modified
+        a_p = pinv(a_m, atol=atol, rtol=0.)
+        adiff1 = a @ a_p @ a - a
+        adiff2 = a_m @ a_p @ a_m - a_m
+        # Now adiff1 should be around atol value while adiff2 should be
+        # relatively tiny
+        assert_allclose(np.linalg.norm(adiff1), 5e-4, atol=5.e-4)
+        assert_allclose(np.linalg.norm(adiff2), 5e-14, atol=5.e-14)
+
+        # Now do the same but remove another sv ~4.234 via rtol
+        a_p = pinv(a_m, atol=atol, rtol=rtol)
+        adiff1 = a @ a_p @ a - a
+        adiff2 = a_m @ a_p @ a_m - a_m
+        assert_allclose(np.linalg.norm(adiff1), 4.233, rtol=0.01)
+        assert_allclose(np.linalg.norm(adiff2), 4.233, rtol=0.01)
+
+
+@pytest.mark.filterwarnings('ignore::DeprecationWarning')
 class TestPinvSymmetric:
 
+    def setup_method(self):
+        np.random.seed(1234)
+
     def test_simple_real(self):
         a = array([[1, 2, 3], [4, 5, 6], [7, 8, 10]], dtype=float)
         a = np.dot(a, a.T)
@@ -1302,20 +1329,35 @@ def test_native_list_argument(self):
         a_pinv = pinvh(a.tolist())
         assert_array_almost_equal(np.dot(a, a_pinv), np.eye(3))
 
-
-def test_pinv_pinv2_comparison():  # As reported in gh-8861
-    I_6 = np.eye(6)
-    Ts = np.diag([-1] * 4 + [-2], k=-1) + np.diag([-2] + [-1] * 4, k=1)
-    T = I_6 + Ts
-    A = 25 * (np.kron(I_6, T) + np.kron(Ts, I_6))
-
-    Ap, Ap2 = pinv(A), pinv2(A)
-
-    tol = 1e-11
-    assert_allclose(A @ Ap @ A - A, A @ Ap2 @ A - A, rtol=0., atol=tol)
-    assert_allclose(Ap @ A @ Ap - Ap, Ap2 @ A @ Ap2 - Ap2, rtol=0., atol=tol)
-
-
+    def test_atol_rtol(self):
+        n = 12
+        # get a random ortho matrix for shuffling
+        q, _ = qr(np.random.rand(n, n))
+        a = np.diag([4, 3, 2, 1, 0.99e-4, 0.99e-5] + [0.99e-6]*(n-6))
+        a = q.T @ a @ q
+        a_m = np.diag([4, 3, 2, 1, 0.99e-4, 0.] + [0.]*(n-6))
+        a_m = q.T @ a_m @ q
+        atol = 1e-5
+        rtol = (4.01e-4 - 4e-5)/4
+        # Just abs cutoff such that we arrive at a_modified
+        a_p = pinvh(a, atol=atol, rtol=0.)
+        adiff1 = a @ a_p @ a - a
+        adiff2 = a_m @ a_p @ a_m - a_m
+        # Now adiff1 should dance around atol value since truncation
+        # while adiff2 should be relatively tiny
+        assert_allclose(norm(adiff1), atol, rtol=0.1)
+        assert_allclose(norm(adiff2), 1e-12, atol=1e-11)
+
+        # Now do the same but through rtol cancelling atol value
+        a_p = pinvh(a, atol=atol, rtol=rtol)
+        adiff1 = a @ a_p @ a - a
+        adiff2 = a_m @ a_p @ a_m - a_m
+        # adiff1 and adiff2 should be elevated to ~1e-4 due to mismatch
+        assert_allclose(norm(adiff1), 1e-4, rtol=0.1)
+        assert_allclose(norm(adiff2), 1e-4, rtol=0.1)
+
+
+@pytest.mark.filterwarnings('ignore::DeprecationWarning')
 @pytest.mark.parametrize('scale', (1e-20, 1., 1e20))
 @pytest.mark.parametrize('pinv_', (pinv, pinvh, pinv2))
 def test_auto_rcond(scale, pinv_):
@@ -1458,6 +1500,7 @@ def test_lstsq(self):
     def test_pinv(self):
         assert_no_overwrite(pinv, [(3, 3)])
 
+    @pytest.mark.filterwarnings('ignore::DeprecationWarning')
     def test_pinv2(self):
         assert_no_overwrite(pinv2, [(3, 3)])