Repository: cupy/cupy

Issue:
possible bug in cupyx sparse matrix transpose/multiplication
Hello cupy(x) developers,

I would like to report what looks like a bug in the cupyx sparse matrix library. I am running on a Skylake cpu and a Tesla V100 gpu on a SUSE-ish Linux system. CuPy version: 6.0.0. CUDA version 10.1.168. I am running inside a conda environment which you can find in my requirements.txt file: 

[requirements.txt](https://github.com/cupy/cupy/files/3474978/requirements.txt)

Here is a reproducer in which I am trying to compare the same operations on the cpu and gpu and checking at every step that they are the same. Finally in the last step the `np.allclose()` check fails. The `iCov_max_diff` value is much larger than machine precision which suggests there is a bug. 

```
import numpy as np
import scipy.sparse
import cupy as cp
import cupyx as cpx

m = n = 1000

np.random.seed(1)

#create random data to use in W
random_data = np.random.random(m)

#create A random sparse matrix
A_cpu = scipy.sparse.random(m, n, format='csr', random_state=42)
A_gpu = cpx.scipy.sparse.csr_matrix(A_cpu)
#yank gpu back and compare
A_yank = A_gpu.get()
assert np.allclose(A_cpu.todense(),A_yank.todense())

#create W random sparse matrix
W_cpu = scipy.sparse.spdiags(data=random_data, diags=[0,], m=m, n=n)
W_gpu = cpx.scipy.sparse.spdiags(data=random_data, diags=[0,], m=m, n=n)
#yank gpu back and compare
W_yank = W_gpu.get()
assert np.allclose(W_cpu.todense(),W_yank.todense())

#see how the dot products go
W_dot_A_cpu = W_cpu.dot(A_cpu)
W_dot_A_gpu = W_gpu.dot(A_gpu)
#yank gpu back and compare
W_dot_A_yank = W_dot_A_gpu.get()
assert np.allclose(W_dot_A_cpu.todense(), W_dot_A_yank.todense())

#check the transpose
A_trans_cpu = A_cpu.T
A_trans_gpu = A_gpu.T
#yank gpu back and compare
A_trans_yank = A_trans_gpu.get() #use get bc its a sparse object
assert np.allclose(A_trans_cpu.todense(), A_trans_yank.todense())

#okay now inverse covariance (where things go wrong)
iCov_cpu = A_cpu.T.dot(W_dot_A_cpu)
iCov_gpu = A_gpu.T.dot(W_dot_A_gpu)
#yank gpu back and compare
iCov_yank = iCov_gpu.get()

iCov_diff = iCov_cpu.todense() - iCov_yank.todense()
max_iCov_diff = np.max(iCov_diff)
print("max iCov diff")
print(max_iCov_diff)

assert np.allclose(iCov_cpu.todense(), iCov_yank.todense()) #fails for large matrix sizes
```

If there is more information I can provide please let me know. Of course if I am doing something wrong I would also be happy for your feedback about how to do this correctly. 

Thank you very much for your help,
Laurie





Discussion:
I identified that the issue comes from `sum_duplicates()`, which is called to convert the sparse matrix to canonical format before `csrgemm` here: https://github.com/cupy/cupy/blob/master/cupyx/scipy/sparse/csr.py#L98

Indeed, by computing `sum_duplicates` of `A_trans_cpu` on CPU and then only copying the transposed matrix (in canonical format) to GPU, the `assert` is successful. So it is likely related to https://github.com/cupy/cupy/issues/2005. Indeed by inspecting the values of the sparse matrices, the error first occurs here https://github.com/cupy/cupy/blob/master/cupyx/scipy/sparse/compressed.py#L337, when the data is converted from coo to csr (after calling `sum_duplicates` on the coo). And the error is fixed by doing the conversion from coo to csr on the CPU instead of GPU.

```
#check the transpose
A_trans_cpu = A_cpu.T
A_trans_cpu.sum_duplicates()
A_trans_gpu = cpx.scipy.sparse.csr_matrix(A_trans_cpu)
```

Resolving patch:
diff --git a/cupy/cusparse.py b/cupy/cusparse.py
--- a/cupy/cusparse.py
+++ b/cupy/cusparse.py
@@ -525,13 +525,14 @@ def csrsort(x):
         x.indices.data.ptr)
     buf = cupy.empty(buffer_size, 'b')
     P = cupy.empty(nnz, 'i')
+    data_orig = x.data.copy()
     cusparse.createIdentityPermutation(handle, nnz, P.data.ptr)
     cusparse.xcsrsort(
         handle, m, n, nnz, x._descr.descriptor, x.indptr.data.ptr,
         x.indices.data.ptr, P.data.ptr, buf.data.ptr)
     _call_cusparse(
         'gthr', x.dtype,
-        handle, nnz, x.data.data.ptr, x.data.data.ptr,
+        handle, nnz, data_orig.data.ptr, x.data.data.ptr,
         P.data.ptr, cusparse.CUSPARSE_INDEX_BASE_ZERO)
 
 
@@ -553,17 +554,24 @@ def cscsort(x):
         x.indices.data.ptr)
     buf = cupy.empty(buffer_size, 'b')
     P = cupy.empty(nnz, 'i')
+    data_orig = x.data.copy()
     cusparse.createIdentityPermutation(handle, nnz, P.data.ptr)
     cusparse.xcscsort(
         handle, m, n, nnz, x._descr.descriptor, x.indptr.data.ptr,
         x.indices.data.ptr, P.data.ptr, buf.data.ptr)
     _call_cusparse(
         'gthr', x.dtype,
-        handle, nnz, x.data.data.ptr, x.data.data.ptr,
+        handle, nnz, data_orig.data.ptr, x.data.data.ptr,
         P.data.ptr, cusparse.CUSPARSE_INDEX_BASE_ZERO)
 
 
 def coosort(x):
+    """Sorts indices of COO-matrix in place.
+
+    Args:
+        x (cupyx.scipy.sparse.coo_matrix): A sparse matrix to sort.
+
+    """
     nnz = x.nnz
     if nnz == 0:
         return
@@ -574,13 +582,14 @@ def coosort(x):
         handle, m, n, nnz, x.row.data.ptr, x.col.data.ptr)
     buf = cupy.empty(buffer_size, 'b')
     P = cupy.empty(nnz, 'i')
+    data_orig = x.data.copy()
     cusparse.createIdentityPermutation(handle, nnz, P.data.ptr)
     cusparse.xcoosortByRow(
         handle, m, n, nnz, x.row.data.ptr, x.col.data.ptr,
         P.data.ptr, buf.data.ptr)
     _call_cusparse(
         'gthr', x.dtype,
-        handle, nnz, x.data.data.ptr, x.data.data.ptr,
+        handle, nnz, data_orig.data.ptr, x.data.data.ptr,
         P.data.ptr, cusparse.CUSPARSE_INDEX_BASE_ZERO)
 
 


Test patch:
diff --git a/tests/cupy_tests/test_cusparse.py b/tests/cupy_tests/test_cusparse.py
--- a/tests/cupy_tests/test_cusparse.py
+++ b/tests/cupy_tests/test_cusparse.py
@@ -193,3 +193,58 @@ def test_csrmvEx_with_y(self):
         expect = self.alpha * self.op_a.dot(self.x) + self.beta * self.y
         self.assertIs(y, z)
         testing.assert_array_almost_equal(y, expect)
+
+
+@testing.with_requires('scipy')
+class TestCoosort(unittest.TestCase):
+
+    def setUp(self):
+        self.a = scipy.sparse.random(
+            100, 100, density=0.9, dtype=numpy.float32, format='coo')
+
+    def test_coosort(self):
+        a = sparse.coo_matrix(self.a)
+        cupy.cusparse.coosort(a)
+        # lexsort by row first and col second
+        argsort = numpy.lexsort((self.a.col, self.a.row))
+        testing.assert_array_equal(self.a.row[argsort], a.row)
+        testing.assert_array_equal(self.a.col[argsort], a.col)
+        testing.assert_array_almost_equal(self.a.data[argsort], a.data)
+
+
+@testing.with_requires('scipy')
+class TestCsrsort(unittest.TestCase):
+
+    def setUp(self):
+        self.a = scipy.sparse.random(
+            1, 1000, density=0.9, dtype=numpy.float32, format='csr')
+        numpy.random.shuffle(self.a.indices)
+        self.a.has_sorted_indices = False
+
+    def test_csrsort(self):
+        a = sparse.csr_matrix(self.a)
+        cupy.cusparse.csrsort(a)
+
+        self.a.sort_indices()
+        testing.assert_array_equal(self.a.indptr, a.indptr)
+        testing.assert_array_equal(self.a.indices, a.indices)
+        testing.assert_array_almost_equal(self.a.data, a.data)
+
+
+@testing.with_requires('scipy')
+class TestCscsort(unittest.TestCase):
+
+    def setUp(self):
+        self.a = scipy.sparse.random(
+            1000, 1, density=0.9, dtype=numpy.float32, format='csc')
+        numpy.random.shuffle(self.a.indices)
+        self.a.has_sorted_indices = False
+
+    def test_csrsort(self):
+        a = sparse.csc_matrix(self.a)
+        cupy.cusparse.cscsort(a)
+
+        self.a.sort_indices()
+        testing.assert_array_equal(self.a.indptr, a.indptr)
+        testing.assert_array_equal(self.a.indices, a.indices)
+        testing.assert_array_almost_equal(self.a.data, a.data)