Repository: pandas-dev/pandas

Issue:
API: flex comparisons DataFrame vs Series inconsistency
In tests.frame.test_arithmetic we test the DataFrame part of the following, but we do not test the Series behavior (at least not in that file)

```
arr = np.array([np.nan, 1, 6, np.nan])
arr2 = np.array([2j, np.nan, 7, None])
ser = pd.Series(arr)
ser2 = pd.Series(arr2)
df = pd.DataFrame(ser)
df2 = pd.DataFrame(ser2)

ser < ser2  # raises TypeError, makes sense
df < df2  # raises TypeError, makes sense

ser.lt(ser2)  # raises TypeError, makes sense

>>> df.lt(df2)
       0
0  False
1  False
2   True
3  False
```

The `df.lt(df2)` version (the "flex" op) masks positions where either `df` or `df2` is null.  That is fine, but why doesn't the `Series` version do the same thing?


Discussion:
@jreback @jorisvandenbossche @TomAugspurger this won't be hard to fix, needs input on what the desired behavior is.
What's the proposal here? To always mask before doing the op, so that `ser.lt(ser2)` doesn't raise?
Two options:

a) make Series.lt behave like DataFrame.lt
b) make DataFrame.lt behave like Series.lt

The main argument for a) is that it is less breaking for users relying on the DataFrame behavior.
The main argument for b) is that the behavior only matters in exactly 1 test, and it is a test with truly weird complex-dtype behavior (#28050)
OK, I'm not sure what's best here. My inclination is to follow NumPy where possible, which I think argues for option a, right? (I'm ignoring the `None`, which forces object dtype. Maybe I shouldn't ignore that).

Resolving patch:
diff --git a/pandas/core/ops/__init__.py b/pandas/core/ops/__init__.py
--- a/pandas/core/ops/__init__.py
+++ b/pandas/core/ops/__init__.py
@@ -28,7 +28,7 @@
     ABCIndexClass,
     ABCSeries,
 )
-from pandas.core.dtypes.missing import isna, notna
+from pandas.core.dtypes.missing import isna
 
 from pandas._typing import ArrayLike
 from pandas.core.construction import array, extract_array
@@ -354,38 +354,6 @@ def fill_binop(left, right, fill_value):
     return left, right
 
 
-def mask_cmp_op(x, y, op):
-    """
-    Apply the function `op` to only non-null points in x and y.
-
-    Parameters
-    ----------
-    x : array-like
-    y : array-like
-    op : binary operation
-
-    Returns
-    -------
-    result : ndarray[bool]
-    """
-    xrav = x.ravel()
-    result = np.empty(x.size, dtype=bool)
-    if isinstance(y, (np.ndarray, ABCSeries)):
-        yrav = y.ravel()
-        mask = notna(xrav) & notna(yrav)
-        result[mask] = op(np.array(list(xrav[mask])), np.array(list(yrav[mask])))
-    else:
-        mask = notna(xrav)
-        result[mask] = op(np.array(list(xrav[mask])), y)
-
-    if op == operator.ne:  # pragma: no cover
-        np.putmask(result, ~mask, True)
-    else:
-        np.putmask(result, ~mask, False)
-    result = result.reshape(x.shape)
-    return result
-
-
 # -----------------------------------------------------------------------------
 # Dispatch logic
 
@@ -905,14 +873,6 @@ def _flex_comp_method_FRAME(cls, op, special):
     op_name = _get_op_name(op, special)
     default_axis = _get_frame_op_default_axis(op_name)
 
-    def na_op(x, y):
-        try:
-            with np.errstate(invalid="ignore"):
-                result = op(x, y)
-        except TypeError:
-            result = mask_cmp_op(x, y, op)
-        return result
-
     doc = _flex_comp_doc_FRAME.format(
         op_name=op_name, desc=_op_descriptions[op_name]["desc"]
     )
@@ -926,16 +886,16 @@ def f(self, other, axis=default_axis, level=None):
             # Another DataFrame
             if not self._indexed_same(other):
                 self, other = self.align(other, "outer", level=level, copy=False)
-            new_data = dispatch_to_series(self, other, na_op, str_rep)
+            new_data = dispatch_to_series(self, other, op, str_rep)
             return self._construct_result(new_data)
 
         elif isinstance(other, ABCSeries):
             return _combine_series_frame(
-                self, other, na_op, fill_value=None, axis=axis, level=level
+                self, other, op, fill_value=None, axis=axis, level=level
             )
         else:
             # in this case we always have `np.ndim(other) == 0`
-            new_data = dispatch_to_series(self, other, na_op)
+            new_data = dispatch_to_series(self, other, op)
             return self._construct_result(new_data)
 
     f.__name__ = op_name


Test patch:
diff --git a/pandas/tests/frame/test_arithmetic.py b/pandas/tests/frame/test_arithmetic.py
--- a/pandas/tests/frame/test_arithmetic.py
+++ b/pandas/tests/frame/test_arithmetic.py
@@ -235,21 +235,46 @@ def _test_seq(df, idx_ser, col_ser):
         rs = df.le(df)
         assert not rs.loc[0, 0]
 
+    def test_bool_flex_frame_complex_dtype(self):
         # complex
         arr = np.array([np.nan, 1, 6, np.nan])
         arr2 = np.array([2j, np.nan, 7, None])
         df = pd.DataFrame({"a": arr})
         df2 = pd.DataFrame({"a": arr2})
-        rs = df.gt(df2)
-        assert not rs.values.any()
+
+        msg = "|".join(
+            [
+                "'>' not supported between instances of '.*' and 'complex'",
+                r"unorderable types: .*complex\(\)",  # PY35
+            ]
+        )
+        with pytest.raises(TypeError, match=msg):
+            # inequalities are not well-defined for complex numbers
+            df.gt(df2)
+        with pytest.raises(TypeError, match=msg):
+            # regression test that we get the same behavior for Series
+            df["a"].gt(df2["a"])
+        with pytest.raises(TypeError, match=msg):
+            # Check that we match numpy behavior here
+            df.values > df2.values
+
         rs = df.ne(df2)
         assert rs.values.all()
 
         arr3 = np.array([2j, np.nan, None])
         df3 = pd.DataFrame({"a": arr3})
-        rs = df3.gt(2j)
-        assert not rs.values.any()
 
+        with pytest.raises(TypeError, match=msg):
+            # inequalities are not well-defined for complex numbers
+            df3.gt(2j)
+        with pytest.raises(TypeError, match=msg):
+            # regression test that we get the same behavior for Series
+            df3["a"].gt(2j)
+        with pytest.raises(TypeError, match=msg):
+            # Check that we match numpy behavior here
+            df3.values > 2j
+
+    def test_bool_flex_frame_object_dtype(self):
         # corner, dtype=object
         df1 = pd.DataFrame({"col": ["foo", np.nan, "bar"]})
         df2 = pd.DataFrame({"col": ["foo", datetime.now(), "bar"]})