Repository: tensorly/tensorly

Issue:
[BUG] singular vectors in `tl.partial_svd` are not orthonormal
When `n_eigenvecs` is greater than the input matrix rank, the returned singular values are not orthonormal. However, the result is still a valid low-rank decomposition (<img src="https://render.githubusercontent.com/render/math?math=A = USV">). 

```python3
import tensorly as tl
import numpy as np
np.random.seed(0)

a = tl.dot(tl.randn((4, 2)), tl.randn((2, 4)))
u, s, v = tl.partial_svd(a, n_eigenvecs=3)
print(tl.dot(u.T, u))
```
```
[[ 1. -0.  0.]
 [-0.  1.  0.]
 [ 0.  0.  0.]]
```
<img src="https://render.githubusercontent.com/render/math?math=U^\top U \neq I"> ☹️.

---
To fix this bug we could use the Gram–Schmidt process and extend the singular vectors basis corresponding to the zero singular values.


Comments:

Patch:
diff --git a/tensorly/backend/core.py b/tensorly/backend/core.py
index ae9beb6..41c483a 100644
--- a/tensorly/backend/core.py
+++ b/tensorly/backend/core.py
@@ -1079,6 +1079,9 @@ class Backend(object):
                 S = np.sqrt(np.clip(S, 0, None))
                 S = np.clip(S, np.finfo(S.dtype).eps, None)  # To avoid divide by zero warning on next line
                 V = np.dot(matrix.T.conj(), U * np.where(np.abs(S) <= np.finfo(S.dtype).eps, 0, 1/S)[None, :])
+                U, S, V = U[:, ::-1], S[::-1], V[:, ::-1]
+                V, R = np.linalg.qr(V)
+                V = V * (2*(np.diag(R) >= 0) - 1)  # we can't use np.sign because np.sign(0) == 0
             else:
                 S, V = scipy.sparse.linalg.eigsh(
                     np.dot(matrix.T.conj(), matrix), k=n_eigenvecs, which='LM', v0=v0
@@ -1086,9 +1089,11 @@ class Backend(object):
                 S = np.sqrt(np.clip(S, 0, None))
                 S = np.clip(S, np.finfo(S.dtype).eps, None)
                 U = np.dot(matrix, V) * np.where(np.abs(S) <= np.finfo(S.dtype).eps, 0, 1/S)[None, :]
+                U, S, V = U[:, ::-1], S[::-1], V[:, ::-1]
+                U, R = np.linalg.qr(U)
+                U = U * (2*(np.diag(R) >= 0) - 1)
 
             # WARNING: here, V is still the transpose of what it should be
-            U, S, V = U[:, ::-1], S[::-1], V[:, ::-1]
             V = V.T.conj()
 
         if flip:
diff --git a/tensorly/backend/pytorch_backend.py b/tensorly/backend/pytorch_backend.py
index 267cda3..b96e7d7 100644
--- a/tensorly/backend/pytorch_backend.py
+++ b/tensorly/backend/pytorch_backend.py
@@ -148,6 +148,10 @@ class PyTorchBackend(Backend):
     def stack(arrays, axis=0):
         return torch.stack(arrays, dim=axis)
 
+    @staticmethod
+    def diag(tensor, k=0):
+        return torch.diag(tensor, diagonal=k)
+
     @staticmethod
     def sort(tensor, axis, descending = False):
         if axis is None:
@@ -212,7 +216,7 @@ class PyTorchBackend(Backend):
 for name in ['float64', 'float32', 'int64', 'int32', 'complex128', 'complex64',
              'is_tensor', 'ones', 'zeros', 'any', 'trace', 'cumsum', 'tensordot',
              'zeros_like', 'reshape', 'eye', 'max', 'min', 'prod', 'abs', 'matmul',
-             'sqrt', 'sign', 'where', 'conj', 'diag', 'finfo', 'einsum', 'log2', 'sin', 'cos']:
+             'sqrt', 'sign', 'where', 'conj', 'finfo', 'einsum', 'log2', 'sin', 'cos']:
     PyTorchBackend.register_method(name, getattr(torch, name))
 
 
diff --git a/tensorly/backend/tensorflow_backend.py b/tensorly/backend/tensorflow_backend.py
index 3da3142..1d89fb5 100644
--- a/tensorly/backend/tensorflow_backend.py
+++ b/tensorly/backend/tensorflow_backend.py
@@ -191,7 +191,7 @@ _FUN_NAMES = [
     (np.complex64, 'complex64'),
     (tf.ones, 'ones'),
     (tf.zeros, 'zeros'),
-    (tf.linalg.tensor_diag, 'diag'),
+    (tf.linalg.diag, 'diag'),
     (tf.zeros_like, 'zeros_like'),
     (tf.eye, 'eye'),
     (tf.reshape, 'reshape'),


Test patch:
diff --git a/tensorly/tests/test_backend.py b/tensorly/tests/test_backend.py
index 04818a3..c73c24f 100644
--- a/tensorly/tests/test_backend.py
+++ b/tensorly/tests/test_backend.py
@@ -161,7 +161,7 @@ def test_svd():
             assert_(left_orthogonality_error <= tol_orthogonality,
                 msg='Left eigenvecs not orthogonal for "{}" svd fun VS svd and backend="{}, for {} eigenenvecs, and size {}".'.format(
                         name, tl.get_backend(), n, s))
-            right_orthogonality_error = T.norm(T.dot(T.transpose(fU), fU) - T.eye(n))
+            right_orthogonality_error = T.norm(T.dot(fV, T.transpose(fV)) - T.eye(n))
             assert_(right_orthogonality_error <= tol_orthogonality,
                 msg='Right eigenvecs not orthogonal for "{}" svd fun VS svd and backend="{}, for {} eigenenvecs, and size {}".'.format(
                     name, tl.get_backend(), n, s))
@@ -180,6 +180,14 @@ def test_svd():
         assert_(np.isfinite(T.to_numpy(U)).all(), msg="Left singular vectors are not finite")
         assert_(np.isfinite(T.to_numpy(V)).all(), msg="Right singular vectors are not finite")
 
+        # Test orthonormality when  max_dim > n_eigenvecs > matrix_rank
+        matrix = tl.dot(tl.randn((4, 2), seed=1), tl.randn((2, 4), seed=12))
+        U, S, V = tl.partial_svd(matrix, n_eigenvecs=3, random_state=0)
+        left_orthogonality_error = T.norm(T.dot(T.transpose(U), U) - T.eye(3))
+        assert_(left_orthogonality_error <= tol_orthogonality)
+        right_orthogonality_error = T.norm(T.dot(V, T.transpose(V)) - T.eye(3))
+        assert_(right_orthogonality_error <= tol_orthogonality)
+
         # Test if partial_svd returns the same result for the same setting
         matrix = T.tensor(np.random.random((20, 5)))
         random_state = np.random.RandomState(0)