Repository: scipy/scipy

Issue:
When using arrays of length longer than max int32 X.sort_indices segfaults
https://github.com/scipy/scipy/blob/03b1092cf0e0bdebcbe98a44d289208a1e597416/scipy/sparse/sputils.py#L153

The line linked will cause a segfault if you pass in uint32 data type arrays as indices and indptr.

```python
>>> import numpy as np
>>> np.ones(4, dtype='uint32')
>>> x = np.ones(4, dtype='uint32')
>>> x.dtype > np.int32
False
>>>
```

```python
# segfault.py
import numpy as np
from scipy.sparse import csc_matrix

data = np.ones(2212191192, dtype='float32')
indices = np.random.random_integers(65793, size=(2212191192,)).astype(np.uint32)
indptr = np.zeros(65794, dtype='uint32')
for i in range(1, 65793):
    indptr[i] = 33623 * i
indptr[-1] = 2212191192
X = csc_matrix((data, indices, indptr), dtype='float32')
X.sort_indices()

# python3 segfault.py
[1]    55277 segmentation fault  python3 segfault.py
```

Expected outcome is to upgrade the uint32 arrays to int64 and proceed without segmentation fault.

Alternatively it would be nice if uint32 could be used as the backing datatype for the arrays so that we can store the 2+ billion integers with half of the space overhead.


Discussion:
The check probably should use `np.can_cast` and not `>`

Yeah :) I was making a PR for it 
Just waiting on the tests to pass 

Resolving patch:
diff --git a/scipy/sparse/sputils.py b/scipy/sparse/sputils.py
--- a/scipy/sparse/sputils.py
+++ b/scipy/sparse/sputils.py
@@ -139,6 +139,7 @@ def get_index_dtype(arrays=(), maxval=None, check_contents=False):
 
     """
 
+    int32min = np.iinfo(np.int32).min
     int32max = np.iinfo(np.int32).max
 
     dtype = np.intc
@@ -151,7 +152,7 @@ def get_index_dtype(arrays=(), maxval=None, check_contents=False):
 
     for arr in arrays:
         arr = np.asarray(arr)
-        if arr.dtype > np.int32:
+        if not np.can_cast(arr.dtype, np.int32):
             if check_contents:
                 if arr.size == 0:
                     # a bigger type not needed
@@ -159,8 +160,7 @@ def get_index_dtype(arrays=(), maxval=None, check_contents=False):
                 elif np.issubdtype(arr.dtype, np.integer):
                     maxval = arr.max()
                     minval = arr.min()
-                    if (minval >= np.iinfo(np.int32).min and
-                            maxval <= np.iinfo(np.int32).max):
+                    if minval >= int32min and maxval <= int32max:
                         # a bigger type not needed
                         continue
 


Test patch:
diff --git a/scipy/sparse/tests/test_sputils.py b/scipy/sparse/tests/test_sputils.py
--- a/scipy/sparse/tests/test_sputils.py
+++ b/scipy/sparse/tests/test_sputils.py
@@ -83,12 +83,67 @@ def test_isdense(self):
         assert_equal(sputils.isdense(np.matrix([1])), True)
 
     def test_validateaxis(self):
-        func = sputils.validateaxis
-
-        assert_raises(TypeError, func, (0, 1))
-        assert_raises(TypeError, func, 1.5)
-        assert_raises(ValueError, func, 3)
+        assert_raises(TypeError, sputils.validateaxis, (0, 1))
+        assert_raises(TypeError, sputils.validateaxis, 1.5)
+        assert_raises(ValueError, sputils.validateaxis, 3)
 
         # These function calls should not raise errors
         for axis in (-2, -1, 0, 1, None):
-            func(axis)
+            sputils.validateaxis(axis)
+
+    def test_get_index_dtype(self):
+        imax = np.iinfo(np.int32).max
+        too_big = imax + 1
+
+        # Check that uint32's with no values too large doesn't return
+        # int64
+        a1 = np.ones(90, dtype='uint32')
+        a2 = np.ones(90, dtype='uint32')
+        assert_equal(
+            np.dtype(sputils.get_index_dtype((a1, a2), check_contents=True)),
+            np.dtype('int32')
+        )
+
+        # Check that if we can not convert but all values are less than or
+        # equal to max that we can just convert to int32
+        a1[-1] = imax
+        assert_equal(
+            np.dtype(sputils.get_index_dtype((a1, a2), check_contents=True)),
+            np.dtype('int32')
+        )
+
+        # Check that if it can not convert directly and the contents are
+        # too large that we return int64
+        a1[-1] = too_big
+        assert_equal(
+            np.dtype(sputils.get_index_dtype((a1, a2), check_contents=True)),
+            np.dtype('int64')
+        )
+
+        # test that if can not convert and didn't specify to check_contents
+        # we return int64
+        a1 = np.ones(89, dtype='uint32')
+        a2 = np.ones(89, dtype='uint32')
+        assert_equal(
+            np.dtype(sputils.get_index_dtype((a1, a2))),
+            np.dtype('int64')
+        )
+
+        # Check that even if we have arrays that can be converted directly
+        # that if we specify a maxval directly it takes precedence
+        a1 = np.ones(12, dtype='uint32')
+        a2 = np.ones(12, dtype='uint32')
+        assert_equal(
+            np.dtype(sputils.get_index_dtype(
+                (a1, a2), maxval=too_big, check_contents=True
+            )),
+            np.dtype('int64')
+        )
+
+        # Check that an array with a too max size and maxval set
+        # still returns int64
+        a1[-1] = too_big
+        assert_equal(
+            np.dtype(sputils.get_index_dtype((a1, a2), maxval=too_big)),
+            np.dtype('int64')
+        )