Repository: scipy/scipy

Issue:
CSR matrix comparison operators broken with duplicate entries
```
>>> M = sp.csr_matrix((np.array([-2, 4]), np.array([0, 0]), np.array([0, 2])))
>>> M.A
array([[2]])
>>> (M > 1).A
array([[False]], dtype=bool)
```

I haven't analysed where the failure occurs, but there is apparently no test for this case at present, as `TestCSRNonCanonical.test_gt` passes.



Discussion:
This applies to all CSR/C comparators, which use logic like https://github.com/scipy/scipy/blob/master/scipy/sparse/compressed.py#L289:

``` python
                other_arr = self.copy()
                other_arr.data[:] = other
                return self._binopt(other_arr, '_gt_')
```

Where there are duplicates, `other` will be multiplied, so tests passing is dependent on the relative values of `self` and `other * 2`.

The simplest solution it to call `self.sum_duplicates()` before the `copy`. But after calling sum_duplicates, I would just return `self._with_data(self.data > other)` anyway.

I can't think of a way to construct `other_arr` that would be faster than `sum_duplicates`. For example, `other_arr = self.__class__(); other_arr[self.nonzero()] = other` requires a sort in `__setitem__`.


Resolving patch:
diff --git a/scipy/sparse/compressed.py b/scipy/sparse/compressed.py
--- a/scipy/sparse/compressed.py
+++ b/scipy/sparse/compressed.py
@@ -4,6 +4,7 @@
 __all__ = []
 
 from warnings import warn
+import operator
 
 import numpy as np
 from scipy.lib.six import xrange, zip as izip
@@ -189,11 +190,26 @@ def check_format(self, full_check=True):
     # Boolean comparisons #
     #######################
 
+    def _copy_with_const(self, const):
+        """Copy data, with all nonzeros replaced with constant for binopt
+
+        Adopts the dtype of const to avoid removing sign or magnitude before
+        comparison.
+
+        Warning: does not make a copy of indices and indptr
+        """
+        try:
+            self.sum_duplicates()
+        except NotImplementedError:
+            pass
+        data = np.empty(self.data.shape, dtype=np.asarray(const).dtype)
+        data.fill(const)
+        return self.__class__((data, self.indices, self.indptr), shape=self.shape)
+
     def __eq__(self, other):
         # Scalar other.
         if isscalarlike(other):
-            other_arr = self.copy()
-            other_arr.data[:] = other
+            other_arr = self._copy_with_const(other)
             res = self._binopt(other_arr,'_ne_')
             if other == 0:
                 warn("Comparing a sparse matrix with 0 using == is inefficient"
@@ -231,8 +247,7 @@ def __ne__(self, other):
                 res = (self == other)
                 return all_true - res
             else:
-                other_arr = self.copy()
-                other_arr.data[:] = other
+                other_arr = self._copy_with_const(other)
                 return self._binopt(other_arr,'_ne_')
         # Dense other.
         elif isdense(other):
@@ -248,83 +263,23 @@ def __ne__(self, other):
         else:
             return True
 
-    def __lt__(self, other):
+    def _inequality(self, other, op, op_name, bad_scalar_msg):
         # Scalar other.
         if isscalarlike(other):
-            if 0 < other:
-                warn("Comparing a sparse matrix with a scalar greater than "
-                     "zero using < is inefficient, try using > instead.", SparseEfficiencyWarning)
-                other_arr = np.empty(self.shape)
-                other_arr.fill(other)
-                other_arr = self.__class__(other_arr)
-                return self._binopt(other_arr, '_lt_')
-            else:
-                other_arr = self.copy()
-                other_arr.data[:] = other
-                return self._binopt(other_arr, '_lt_')
-        # Dense other.
-        elif isdense(other):
-            return self.todense() < other
-        # Sparse other.
-        elif isspmatrix(other):
-            #TODO sparse broadcasting
-            if self.shape != other.shape:
-                raise ValueError("inconsistent shapes")
-            elif self.format != other.format:
-                other = other.asformat(self.format)
-            return self._binopt(other, '_lt_')
-        else:
-            raise ValueError("Operands could not be compared.")
-
-    def __gt__(self, other):
-        # Scalar other.
-        if isscalarlike(other):
-            if 0 > other:
-                warn("Comparing a sparse matrix with a scalar less than zero "
-                     "using > is inefficient, try using < instead.", SparseEfficiencyWarning)
-                other_arr = np.empty(self.shape)
-                other_arr.fill(other)
-                other_arr = self.__class__(other_arr)
-                return self._binopt(other_arr, '_gt_')
-            else:
-                other_arr = self.copy()
-                other_arr.data[:] = other
-                return self._binopt(other_arr, '_gt_')
-        # Dense other.
-        elif isdense(other):
-            return self.todense() > other
-        # Sparse other.
-        elif isspmatrix(other):
-            #TODO sparse broadcasting
-            if self.shape != other.shape:
-                raise ValueError("inconsistent shapes")
-            elif self.format != other.format:
-                other = other.asformat(self.format)
-            return self._binopt(other, '_gt_')
-        else:
-            raise ValueError("Operands could not be compared.")
-
-    def __le__(self,other):
-        # Scalar other.
-        if isscalarlike(other):
-            if 0 == other:
+            if 0 == other and op_name in ('_le_', '_ge_'):
                 raise NotImplementedError(" >= and <= don't work with 0.")
-            elif 0 <= other:
-                warn("Comparing a sparse matrix with a scalar less than zero "
-                     "using <= is inefficient, try using < instead.", SparseEfficiencyWarning)
-                other_arr = np.empty(self.shape)
+            elif op(0, other):
+                warn(bad_scalar_msg, SparseEfficiencyWarning)
+                other_arr = np.empty(self.shape, dtype=np.asarray(other).dtype)
                 other_arr.fill(other)
                 other_arr = self.__class__(other_arr)
-                return self._binopt(other_arr, '_le_')
+                return self._binopt(other_arr, op_name)
             else:
-                # Casting as other's type avoids corner case like
-                # ``spmatrix(True) < -2'' from being True.
-                other_arr = self.astype(type(other)).copy()
-                other_arr.data[:] = other
-                return self._binopt(other_arr, '_le_')
+                other_arr = self._copy_with_const(other)
+                return self._binopt(other_arr, op_name)
         # Dense other.
         elif isdense(other):
-            return self.todense() <= other
+            return op(self.todense(), other)
         # Sparse other.
         elif isspmatrix(other):
             #TODO sparse broadcasting
@@ -332,47 +287,39 @@ def __le__(self,other):
                 raise ValueError("inconsistent shapes")
             elif self.format != other.format:
                 other = other.asformat(self.format)
+            if op_name not in ('_ge_', '_le_'):
+                return self._binopt(other, op_name)
+
             warn("Comparing sparse matrices using >= and <= is inefficient, "
                  "using <, >, or !=, instead.", SparseEfficiencyWarning)
             all_true = self.__class__(np.ones(self.shape))
-            res = self._binopt(other, '_gt_')
+            res = self._binopt(other, '_gt_' if op_name == '_le_' else '_lt_')
             return all_true - res
         else:
             raise ValueError("Operands could not be compared.")
 
+    def __lt__(self, other):
+        return self._inequality(other, operator.lt, '_lt_',
+                                "Comparing a sparse matrix with a scalar "
+                                "greater than zero using < is inefficient, "
+                                "try using >= instead.")
+
+    def __gt__(self, other):
+        return self._inequality(other, operator.gt, '_gt_',
+                                "Comparing a sparse matrix with a scalar "
+                                "less than zero using > is inefficient, "
+                                "try using <= instead.")
+
+    def __le__(self, other):
+        return self._inequality(other, operator.le, '_le_',
+                                "Comparing a sparse matrix with a scalar "
+                                "greater than zero using <= is inefficient, "
+                                "try using > instead.")
     def __ge__(self,other):
-        # Scalar other.
-        if isscalarlike(other):
-            if 0 == other:
-                raise NotImplementedError(" >= and <= don't work with 0.")
-            elif 0 >= other:
-                warn("Comparing a sparse matrix with a scalar greater than zero"
-                     " using >= is inefficient, try using < instead.", SparseEfficiencyWarning)
-                other_arr = np.empty(self.shape)
-                other_arr.fill(other)
-                other_arr = self.__class__(other_arr)
-                return self._binopt(other_arr, '_ge_')
-            else:
-                other_arr = self.astype(type(other)).copy()
-                other_arr.data[:] = other
-                return self._binopt(other_arr, '_ge_')
-        # Dense other.
-        elif isdense(other):
-            return self.todense() >= other
-        # Sparse other.
-        elif isspmatrix(other):
-            #TODO sparse broadcasting
-            if self.shape != other.shape:
-                raise ValueError("inconsistent shapes")
-            elif self.format != other.format:
-                other = other.asformat(self.format)
-            warn("Comparing sparse matrices using >= and <= is inefficient, "
-                 "try using <, >, or !=, instead.", SparseEfficiencyWarning)
-            all_true = self.__class__(np.ones(self.shape))
-            res = self._binopt(other, '_lt_')
-            return all_true - res
-        else:
-            raise ValueError("Operands could not be compared.")
+        return self._inequality(other, operator.ge, '_ge_',
+                                "Comparing a sparse matrix with a scalar "
+                                "less than zero using >= is inefficient, "
+                                "try using < instead.")
 
     #################################
     # Arithmatic operator overrides #


Test patch:
diff --git a/scipy/sparse/tests/test_base.py b/scipy/sparse/tests/test_base.py
--- a/scipy/sparse/tests/test_base.py
+++ b/scipy/sparse/tests/test_base.py
@@ -117,7 +117,7 @@ def check(dtype):
             assert_(self.spmatrix([1]))
             assert_(not self.spmatrix([0]))
         for dtype in self.checked_dtypes:
-            fails = self.__class__ == TestDOK
+            fails = isinstance(self, TestDOK)
             msg = "Cannot create a rank <= 2 DOK matrix."
             yield dec.skipif(fails, msg)(check), dtype
 
@@ -158,8 +158,7 @@ def check(dtype):
             assert_array_equal(dat == 1, (datsp == 1).todense())
 
         msg = "Bool comparisons only implemented for BSR, CSC, and CSR."
-        fails = not (self.__class__ == TestBSR or self.__class__ == TestCSC or
-                     self.__class__ == TestCSR)
+        fails = not isinstance(self, (TestBSR, TestCSC, TestCSR))
         for dtype in self.checked_dtypes:
             with warnings.catch_warnings():
                 warnings.simplefilter("ignore", category=np.ComplexWarning)
@@ -194,8 +193,7 @@ def check(dtype):
             assert_array_equal(1 != dat, (1 != datsp).todense())
 
         msg = "Bool comparisons only implemented for BSR, CSC, and CSR."
-        fails = not (self.__class__ == TestBSR or self.__class__ == TestCSC or
-                     self.__class__ == TestCSR)
+        fails = not isinstance(self, (TestBSR, TestCSC, TestCSR))
         for dtype in self.checked_dtypes:
             with warnings.catch_warnings():
                 warnings.simplefilter("ignore", category=np.ComplexWarning)
@@ -259,8 +257,7 @@ def check_fail(dtype):
             assert_array_equal(dat < datsp2, datsp < dat2)
 
         msg = "Bool comparisons only implemented for BSR, CSC, and CSR."
-        fails = not (self.__class__ == TestBSR or self.__class__ == TestCSC or
-                     self.__class__ == TestCSR)
+        fails = not isinstance(self, (TestBSR, TestCSC, TestCSR))
         for dtype in self.checked_dtypes:
             with warnings.catch_warnings():
                 warnings.simplefilter("ignore", category=np.ComplexWarning)
@@ -330,8 +327,7 @@ def check_fail(dtype):
             assert_array_equal(dat > datsp2, datsp > dat2)
 
         msg = "Bool comparisons only implemented for BSR, CSC, and CSR."
-        fails = not (self.__class__ == TestBSR or self.__class__ == TestCSC or
-                     self.__class__ == TestCSR)
+        fails = not isinstance(self, (TestBSR, TestCSC, TestCSR))
         for dtype in self.checked_dtypes:
             with warnings.catch_warnings():
                 warnings.simplefilter("ignore", category=np.ComplexWarning)
@@ -399,8 +395,7 @@ def check_fail(dtype):
             assert_array_equal(dat <= datsp2, datsp <= dat2)
 
         msg = "Bool comparisons only implemented for BSR, CSC, and CSR."
-        fails = not (self.__class__ == TestBSR or self.__class__ == TestCSC or
-                     self.__class__ == TestCSR)
+        fails = not isinstance(self, (TestBSR, TestCSC, TestCSR))
         for dtype in self.checked_dtypes:
             with warnings.catch_warnings():
                 warnings.simplefilter("ignore", category=np.ComplexWarning)
@@ -469,8 +464,7 @@ def check_fail(dtype):
             assert_array_equal(dat >= datsp2, datsp >= dat2)
 
         msg = "Bool comparisons only implemented for BSR, CSC, and CSR."
-        fails = not (self.__class__ == TestBSR or self.__class__ == TestCSC or
-                     self.__class__ == TestCSR)
+        fails = not isinstance(self, (TestBSR, TestCSC, TestCSR))
         for dtype in self.checked_dtypes:
             with warnings.catch_warnings():
                 warnings.simplefilter("ignore", category=np.ComplexWarning)
@@ -841,8 +835,7 @@ def check(dtype):
 
         for dtype in self.checked_dtypes:
             fails = ((dtype == np.typeDict['int']) and
-                    (self.__class__ == TestLIL or
-                     self.__class__ == TestDOK))
+                     isinstance(self, (TestLIL, TestDOK)))
             msg = "LIL and DOK type's __rmul__ method has problems with int data."
             yield dec.knownfailureif(fails, msg)(check), dtype
 
@@ -3469,6 +3462,30 @@ def test_diagonal(self):
     def test_expm(self):
         pass
 
+    @dec.knownfailureif(True, 'inequalities require sum_duplicates, not implemented for BSR')
+    def test_eq(self):
+        pass
+
+    @dec.knownfailureif(True, 'inequalities require sum_duplicates, not implemented for BSR')
+    def test_ne(self):
+        pass
+
+    @dec.knownfailureif(True, 'inequalities require sum_duplicates, not implemented for BSR')
+    def test_gt(self):
+        pass
+
+    @dec.knownfailureif(True, 'inequalities require sum_duplicates, not implemented for BSR')
+    def test_lt(self):
+        pass
+
+    @dec.knownfailureif(True, 'inequalities require sum_duplicates, not implemented for BSR')
+    def test_ge(self):
+        pass
+
+    @dec.knownfailureif(True, 'inequalities require sum_duplicates, not implemented for BSR')
+    def test_le(self):
+        pass
+
 
 class TestCOONonCanonical(_NonCanonicalMixin, TestCOO):
     def _arg1_for_noncanonical(self, M):