Repository: scipy/scipy

Issue:
Integer overflow when concatenating small sparse matrices to large one
https://github.com/scipy/scipy/blob/ad4bf5d439668d2f2ce12c0ff56611777c80f880/scipy/sparse/construct.py#L416

I was concatenating csr matrices row wise using `sparse.vstack` which calls `_compressed_sparse_stack` in this case. That function concatenates the data blocks and column indices (row indices for csc) per matrix and adds up + concatenates the row (column) index pointers. The latter is done using the initial integer type (in my case np.int32) even if the total number of elements is bigger than the integer types max value, causing an integer overflow.

SideNote: This only appears for the matrix (in the block of matrices being concatenated) where the overflow appears in the sequences, as for matrices afterwards the overflow is caught.

There are multiple ways of fixing the issue, I assume determining the dtype of the index pointer automatically by the number of nonzero elements being concatenated is the most favorable one.


Discussion:

Resolving patch:
diff --git a/scipy/sparse/construct.py b/scipy/sparse/construct.py
--- a/scipy/sparse/construct.py
+++ b/scipy/sparse/construct.py
@@ -404,19 +404,25 @@ def _compressed_sparse_stack(blocks, axis):
     """
     other_axis = 1 if axis == 0 else 0
     data = np.concatenate([b.data for b in blocks])
-    indices = np.concatenate([b.indices for b in blocks])
-    indptr = []
-    last_indptr = 0
     constant_dim = blocks[0].shape[other_axis]
+    idx_dtype = get_index_dtype(arrays=[b.indptr for b in blocks],
+                                maxval=max(data.size, constant_dim))
+    indices = np.empty(data.size, dtype=idx_dtype)
+    indptr = np.empty(sum(b.shape[axis] for b in blocks) + 1, dtype=idx_dtype)
+    last_indptr = idx_dtype(0)
     sum_dim = 0
+    sum_indices = 0
     for b in blocks:
         if b.shape[other_axis] != constant_dim:
             raise ValueError('incompatible dimensions for axis %d' % other_axis)
+        indices[sum_indices:sum_indices+b.indices.size] = b.indices
+        sum_indices += b.indices.size
+        idxs = slice(sum_dim, sum_dim + b.shape[axis])
+        indptr[idxs] = b.indptr[:-1]
+        indptr[idxs] += last_indptr
         sum_dim += b.shape[axis]
-        indptr.append(b.indptr[:-1] + last_indptr)
         last_indptr += b.indptr[-1]
-    indptr.append([last_indptr])
-    indptr = np.concatenate(indptr)
+    indptr[-1] = last_indptr
     if axis == 0:
         return csr_matrix((data, indices, indptr),
                           shape=(sum_dim, constant_dim))


Test patch:
diff --git a/scipy/sparse/tests/test_construct.py b/scipy/sparse/tests/test_construct.py
--- a/scipy/sparse/tests/test_construct.py
+++ b/scipy/sparse/tests/test_construct.py
@@ -6,7 +6,9 @@
 from numpy import array, matrix
 from numpy.testing import (assert_equal, assert_,
         assert_array_equal, assert_raises, assert_array_almost_equal_nulp)
+import pytest
 from scipy._lib._numpy_compat import assert_raises_regex
+from scipy._lib._testutils import check_free_memory
 
 from scipy.sparse import csr_matrix, coo_matrix
 
@@ -320,6 +322,10 @@ def test_vstack(self):
                      expected)
         assert_equal(construct.vstack([A.tocsr(),B.tocsr()], dtype=np.float32).dtype,
                      np.float32)
+        assert_equal(construct.vstack([A.tocsr(),B.tocsr()],
+                                      dtype=np.float32).indices.dtype, np.int32)
+        assert_equal(construct.vstack([A.tocsr(),B.tocsr()],
+                                      dtype=np.float32).indptr.dtype, np.int32)
 
     def test_hstack(self):
 
@@ -373,6 +379,20 @@ def test_bmat(self):
                             r'Got blocks\[0,1\]\.shape\[0\] == 1, expected 2',
                             construct.bmat, [[A,C]])
 
+    @pytest.mark.slow
+    def test_concatenate_int32_overflow(self):
+        """ test for indptr overflow when concatenating matrices """
+        check_free_memory(30000)
+        
+        n = 33000
+        A = csr_matrix(np.ones((n, n), dtype=bool))
+        B = A.copy()
+        C = construct._compressed_sparse_stack((A,B), 0)
+        
+        assert_(np.all(np.equal(np.diff(C.indptr), n)))
+        assert_equal(C.indices.dtype, np.int64)
+        assert_equal(C.indptr.dtype, np.int64)
+
     def test_block_diag_basic(self):
         """ basic test for block_diag """
         A = coo_matrix([[1,2],[3,4]])