Repository: scipy/scipy

Issue:
scipy.linalg.pinv gives wrong result while scipy.linalg.pinv2 works
For a student project with finite differences we use the pseudo inverse. Here, we noticed large differences in the accuracy of `scipy.linalg.pinv` and `scipy.linalg.pinv2`: while the latter works just fine, the first produces useless pseudo inverses in this example, see the code below.

As far as I understand, `pinv` uses `scipy.linalg.lstsq` to minimize | e^i - A x^i |_2 for the i-th unit vector e^i in order to obtain the pseudo inverse X, whereas `pinv` directly uses a truncated singular value decomposition.
However, internally `lstsq` uses the LAPACK routine `gelss` which solves the minimization also using a singular value decomposition, see e. g. http://www.netlib.org/lapack/lug/node27.html .

The reason why `pinv` fails for the example below while `pinv2` doesn't results from the fact, that the standard cut-off threshold for the small singular values in `pinv2` is set to `1e6*eps` (for double), whereas `pinv` uses `eps*max(n,m)` if `A` is an `n`x`m` matrix, which is much smaller for the matrix in our example.

Now I wonder why `pinv` and `pinv2` use different cut-off thresholds as standard values. If I use the standard value of `pinv2` for `pinv`, see comment in example code, the results are about the same.
Moreover, I wonder why the least squares solver is namend `pinv` and the *classical* method is called `pinv2`. In my opinion it might by better to switch them.
Since in the end, both use a truncated singular value decomposition, I do not completely understand the fundamental need of both.

### Reproducing code example:
```
import numpy as np
import scipy.sparse as sp
import scipy.linalg as la

ns = 4,
hs = [1 / (n + 1) for n in ns]

def get_A_h(n):
    T_diagonals = [[4] * (n + 2), [-2] + [-1] * n, [-1] * n + [-2]]
    T = sp.diags(T_diagonals, [0, 1, -1])   
    Ts_diagonals = [[-2] + [-1] * n, [-1] * n + [-2]]
    Ts = sp.diags(Ts_diagonals, [1, -1])    
    I_n2 = np.identity(n + 2)
    A_h = (n + 1) ** 2 * (sp.kron(I_n2, T) + sp.kron(Ts, I_n2))
    return A_h

def check_moore_penrose_axioms(A, Apinv):
    print()
    print("1:", la.norm(A@Apinv@A - A))
    print("2:", la.norm(Apinv@A@Apinv - Apinv))
    print("3:", la.norm(A@Apinv - (A@Apinv).T))
    print("4:", la.norm(Apinv@A - (Apinv@A).T))

for n in ns:
    N = n + 2

    A_h = get_A_h(n).toarray()
    R = np.eye(*A_h.shape)
    
    print(A_h)
    
    A_h_plus = la.pinv(A_h) #, rcond=1e6*np.finfo(A_h.dtype).eps)    
    check_moore_penrose_axioms(A_h, A_h_plus)
    
    A_h_plus = la.pinv2(A_h, )
    check_moore_penrose_axioms(A_h, A_h_plus)
```

### Output:
```
[[100. -50.   0. ...   0.   0.   0.]
 [-25. 100. -25. ...   0.   0.   0.]
 [  0. -25. 100. ...   0.   0.   0.]
 ...
 [  0.   0.   0. ... 100. -25.   0.]
 [  0.   0.   0. ... -25. 100. -25.]
 [  0.   0.   0. ...   0. -50. 100.]]

1: 10.966687766642307
2: 3108066184397.3657
3: 0.313565155535093
4: 2.715716258843298

1: 1.2034095577848958e-12
2: 4.613607271655997e-16
3: 2.7619937032685725e-14
4: 2.531755927761354e-14
```


### Scipy/Numpy/Python version information:
```
1.0.1 1.14.3 sys.version_info(major=3, minor=6, micro=5, releaselevel='final', serial=0)
```




Discussion:

Resolving patch:
diff --git a/scipy/linalg/basic.py b/scipy/linalg/basic.py
--- a/scipy/linalg/basic.py
+++ b/scipy/linalg/basic.py
@@ -1256,11 +1256,16 @@ def pinv(a, cond=None, rcond=None, return_rank=False, check_finite=True):
     a : (M, N) array_like
         Matrix to be pseudo-inverted.
     cond, rcond : float, optional
-        Cutoff for 'small' singular values in the least-squares solver.
-        Singular values smaller than ``rcond * largest_singular_value``
-        are considered zero.
-        If None, it is set to ``np.finfo(a.dtype).eps``.
-        If `a` is an array of integers, it is set to ``np.finfo('float64').eps``.
+        Cutoff factor for 'small' singular values. In `lstsq`, 
+        singular values less than ``cond*largest_singular_value`` will be
+        considered as zero. If both are omitted, the default value
+        ``max(M, N) * eps`` is passed to `lstsq` where ``eps`` is the
+        corresponding machine precision value of the datatype of ``a``.
+
+        .. versionchanged:: 1.3.0
+            Previously the default cutoff value was just `eps` without the
+            factor ``max(M, N)``.
+
     return_rank : bool, optional
         if True, return the effective rank of the matrix
     check_finite : bool, optional
@@ -1293,9 +1298,13 @@ def pinv(a, cond=None, rcond=None, return_rank=False, check_finite=True):
     """
     a = _asarray_validated(a, check_finite=check_finite)
     b = np.identity(a.shape[0], dtype=a.dtype)
+
     if rcond is not None:
         cond = rcond
 
+    if cond is None:
+        cond = max(a.shape) * np.spacing(a.real.dtype.type(1))
+
     x, resids, rank, s = lstsq(a, b, cond=cond, check_finite=False)
 
     if return_rank:
@@ -1317,12 +1326,15 @@ def pinv2(a, cond=None, rcond=None, return_rank=False, check_finite=True):
     a : (M, N) array_like
         Matrix to be pseudo-inverted.
     cond, rcond : float or None
-        Cutoff for 'small' singular values.
-        Singular values smaller than ``rcond*largest_singular_value``
-        are considered zero.
-        If None and the dtype of `a` is ``np.float32``, it is set to
-        ``np.finfo('float32').eps * 1e3``.
-        Otherwise, it is set to ``np.finfo('float64').eps * 1e6``.
+        Cutoff for 'small' singular values; singular values smaller than this
+        value are considered as zero. If both are omitted, the default value
+        ``max(M,N)*largest_singular_value*eps`` is used where ``eps`` is the
+        machine precision value of the datatype of ``a``.
+
+        .. versionchanged:: 1.3.0
+            Previously the default cutoff value was just ``eps*f`` where ``f``
+            was ``1e3`` for single precision and ``1e6`` for double precision.
+
     return_rank : bool, optional
         If True, return the effective rank of the matrix.
     check_finite : bool, optional
@@ -1360,10 +1372,9 @@ def pinv2(a, cond=None, rcond=None, return_rank=False, check_finite=True):
         cond = rcond
     if cond in [None, -1]:
         t = u.dtype.char.lower()
-        factor = {'f': 1E3, 'd': 1E6}
-        cond = factor[t] * np.finfo(t).eps
+        cond = np.max(s) * max(a.shape) * np.finfo(t).eps
 
-    rank = np.sum(s > cond * np.max(s))
+    rank = np.sum(s > cond)
 
     u = u[:, :rank]
     u /= s[:rank]
@@ -1389,12 +1400,15 @@ def pinvh(a, cond=None, rcond=None, lower=True, return_rank=False,
     a : (N, N) array_like
         Real symmetric or complex hermetian matrix to be pseudo-inverted
     cond, rcond : float or None
-        Cutoff for 'small' singular values.
-        Singular values smaller than ``rcond*largest_singular_value``
-        are considered zero.
-        If None and the dtype of `a` is ``np.float32``, it is set to
-        ``np.finfo('float32').eps * 1e3``.
-        Otherwise, it is set to ``np.finfo('float64').eps * 1e6``.
+        Cutoff for 'small' singular values; singular values smaller than this
+        value are considered as zero. If both are omitted, the default
+        ``max(M,N)*largest_eigenvalue*eps`` is used where ``eps`` is the
+        machine precision value of the datatype of ``a``.
+
+        .. versionchanged:: 1.3.0
+            Previously the default cutoff value was just ``eps*f`` where ``f``
+            was ``1e3`` for single precision and ``1e6`` for double precision.
+
     lower : bool, optional
         Whether the pertinent array data is taken from the lower or upper
         triangle of `a`. (Default: lower)
@@ -1436,11 +1450,10 @@ def pinvh(a, cond=None, rcond=None, lower=True, return_rank=False,
         cond = rcond
     if cond in [None, -1]:
         t = u.dtype.char.lower()
-        factor = {'f': 1E3, 'd': 1E6}
-        cond = factor[t] * np.finfo(t).eps
+        cond = np.max(np.abs(s)) * max(a.shape) * np.finfo(t).eps
 
     # For Hermitian matrices, singular values equal abs(eigenvalues)
-    above_cutoff = (abs(s) > cond * np.max(abs(s)))
+    above_cutoff = (abs(s) > cond)
     psigma_diag = 1.0 / s[above_cutoff]
     u = u[:, above_cutoff]
 


Test patch:
diff --git a/scipy/linalg/tests/test_basic.py b/scipy/linalg/tests/test_basic.py
--- a/scipy/linalg/tests/test_basic.py
+++ b/scipy/linalg/tests/test_basic.py
@@ -1279,6 +1279,28 @@ def test_native_list_argument(self):
         assert_array_almost_equal(np.dot(a, a_pinv), np.eye(3))
 
 
+def test_pinv_pinv2_comparison():  # As reported in gh-8861
+    I_6 = np.eye(6)
+    Ts = np.diag([-1] * 4 + [-2], k=-1) + np.diag([-2] + [-1] * 4, k=1)
+    T = I_6 + Ts
+    A = 25 * (np.kron(I_6, T) + np.kron(Ts, I_6))
+
+    Ap, Ap2 = pinv(A), pinv2(A)
+
+    tol = 1e-11
+    assert_allclose(A @ Ap @ A - A, A @ Ap2 @ A - A, rtol=0., atol=tol)
+    assert_allclose(Ap @ A @ Ap - Ap, Ap2 @ A @ Ap2 - Ap2, rtol=0., atol=tol)
+
+
+@pytest.mark.parametrize('scale', (1e-20, 1., 1e20))
+@pytest.mark.parametrize('pinv_', (pinv, pinvh, pinv2))
+def test_auto_rcond(scale, pinv_):
+    x = np.array([[1, 0], [0, 1e-10]]) * scale
+    expected = np.diag(1. / np.diag(x))
+    x_inv = pinv_(x)
+    assert_allclose(x_inv, expected)
+
+
 class TestVectorNorms(object):
 
     def test_types(self):