Repository: modin-project/modin

Issue:
BUG: binary operations with custom classes raise errors
### Modin version checks

- [X] I have checked that this issue has not already been reported.

- [X] I have confirmed this bug exists on the latest released version of Modin.

- [x] I have confirmed this bug exists on the main branch of Modin. (In order to do this you can follow [this guide](https://modin.readthedocs.io/en/stable/getting_started/installation.html#installing-from-the-github-master-branch).)


### Reproducible Example

```python
import modin.pandas as pd
import pandas

class CustomNumber():
  def __init__(self, value: int):
    self.value = value
  def __add__(self, other):
    return self.value + other
  def __radd__(self, other):
    return other + self.value

# pandas works without error and gives series of [5]
print(pandas.Series([1]) + CustomNumber(4))
# modin raises error
print(pd.Series([1]) + CustomNumber(4))
```


### Issue Description

`other` is not a scalar in `_validate_other` [here](https://github.com/modin-project/modin/blob/fd776a5a4faf7695d461f369bb2470dcb8aa2745/modin/pandas/base.py#L249), so we check dtype [here](https://github.com/modin-project/modin/blob/fd776a5a4faf7695d461f369bb2470dcb8aa2745/modin/pandas/base.py#L307) and raise `TypeError`.

### Expected Behavior

Can add custom classes that overload addition operators to pandas objects.

### Error Logs

<details>

```python-traceback

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
Input In [1], in <cell line: 15>()
     13 print(pandas.Series([1]) + CustomNumber(4))
     14 # raises error
---> 15 print(pd.Series([1]) + CustomNumber(4))

File ~/software_sources/modin/modin/logging/logger_decorator.py:128, in enable_logging.<locals>.decorator.<locals>.run_and_log(*args, **kwargs)
    113 """
    114 Compute function with logging if Modin logging is enabled.
    115
   (...)
    125 Any
    126 """
    127 if LogMode.get() == "disable":
--> 128     return obj(*args, **kwargs)
    130 logger = get_logger()
    131 logger_level = getattr(logger, log_level)

File ~/software_sources/modin/modin/pandas/series.py:172, in Series.__add__(self, right)
    170 @_doc_binary_op(operation="addition", bin_op="add")
    171 def __add__(self, right):
--> 172     return self.add(right)

File ~/software_sources/modin/modin/logging/logger_decorator.py:128, in enable_logging.<locals>.decorator.<locals>.run_and_log(*args, **kwargs)
    113 """
    114 Compute function with logging if Modin logging is enabled.
    115
   (...)
    125 Any
    126 """
    127 if LogMode.get() == "disable":
--> 128     return obj(*args, **kwargs)
    130 logger = get_logger()
    131 logger_level = getattr(logger, log_level)

File ~/software_sources/modin/modin/pandas/series.py:495, in Series.add(self, other, level, fill_value, axis)
    491 """
    492 Return Addition of series and other, element-wise (binary operator add).
    493 """
    494 new_self, new_other = self._prepare_inter_op(other)
--> 495 return super(Series, new_self).add(
    496     new_other, level=level, fill_value=fill_value, axis=axis
    497 )

File ~/software_sources/modin/modin/logging/logger_decorator.py:128, in enable_logging.<locals>.decorator.<locals>.run_and_log(*args, **kwargs)
    113 """
    114 Compute function with logging if Modin logging is enabled.
    115
   (...)
    125 Any
    126 """
    127 if LogMode.get() == "disable":
--> 128     return obj(*args, **kwargs)
    130 logger = get_logger()
    131 logger_level = getattr(logger, log_level)

File ~/software_sources/modin/modin/pandas/base.py:571, in BasePandasDataset.add(self, other, axis, level, fill_value)
    565 def add(
    566     self, other, axis="columns", level=None, fill_value=None
    567 ):  # noqa: PR01, RT01, D200
    568     """
    569     Return addition of `BasePandasDataset` and `other`, element-wise (binary operator `add`).
    570     """
--> 571     return self._binary_op(
    572         "add", other, axis=axis, level=level, fill_value=fill_value
    573     )

File ~/software_sources/modin/modin/logging/logger_decorator.py:128, in enable_logging.<locals>.decorator.<locals>.run_and_log(*args, **kwargs)
    113 """
    114 Compute function with logging if Modin logging is enabled.
    115
   (...)
    125 Any
    126 """
    127 if LogMode.get() == "disable":
--> 128     return obj(*args, **kwargs)
    130 logger = get_logger()
    131 logger_level = getattr(logger, log_level)

File ~/software_sources/modin/modin/pandas/base.py:396, in BasePandasDataset._binary_op(self, op, other, **kwargs)
    392     kwargs.pop("broadcast", None)
    393     return self._default_to_pandas(
    394         getattr(self._pandas_class, op), other, **kwargs
    395     )
--> 396 other = self._validate_other(other, axis, dtype_check=True)
    397 exclude_list = [
    398     "__add__",
    399     "__radd__",
   (...)
    405     "__rxor__",
    406 ]
    407 if op in exclude_list:

File ~/software_sources/modin/modin/logging/logger_decorator.py:128, in enable_logging.<locals>.decorator.<locals>.run_and_log(*args, **kwargs)
    113 """
    114 Compute function with logging if Modin logging is enabled.
    115
   (...)
    125 Any
    126 """
    127 if LogMode.get() == "disable":
--> 128     return obj(*args, **kwargs)
    130 logger = get_logger()
    131 logger_level = getattr(logger, log_level)

File ~/software_sources/modin/modin/pandas/base.py:317, in BasePandasDataset._validate_other(self, other, axis, dtype_check, compare_index)
    299         self_dtypes = [
    300             dtype
    301             for label, dtype in zip(
   (...)
    304             if label in other
    305         ]
    307     if not all(
    308         (is_numeric_dtype(self_dtype) and is_numeric_dtype(other_dtype))
    309         or (is_object_dtype(self_dtype) and is_object_dtype(other_dtype))
   (...)
    315         for self_dtype, other_dtype in zip(self_dtypes, other_dtypes)
    316     ):
--> 317         raise TypeError("Cannot do operation with improper dtypes")
    318 return result

TypeError: Cannot do operation with improper dtypes


```

</details>


### Installed Versions

<details>

INSTALLED VERSIONS
------------------
commit           : fd776a5a4faf7695d461f369bb2470dcb8aa2745
python           : 3.8.13.final.0
python-bits      : 64
OS               : Darwin
OS-release       : 21.5.0
Version          : Darwin Kernel Version 21.5.0: Tue Apr 26 21:08:22 PDT 2022; root:xnu-8020.121.3~4/RELEASE_X86_64
machine          : x86_64
processor        : i386
byteorder        : little
LC_ALL           : None
LANG             : en_US.UTF-8
LOCALE           : en_US.UTF-8

Modin dependencies
------------------
modin            : 0.17.0+9.gfd776a5a
ray              : 2.0.1
dask             : 2022.01.1
distributed      : 2022.01.1
hdk              : None

pandas dependencies
-------------------
pandas           : 1.5.1
numpy            : 1.23.1
pytz             : 2022.1
dateutil         : 2.8.2
setuptools       : 61.2.0
pip              : 22.1.2
Cython           : None
pytest           : 7.1.2
hypothesis       : None
sphinx           : 5.1.1
blosc            : None
feather          : 0.4.1
xlsxwriter       : None
lxml.etree       : 4.9.1
html5lib         : None
pymysql          : None
psycopg2         : None
jinja2           : 3.1.2
IPython          : 8.4.0
pandas_datareader: None
bs4              : 4.11.1
bottleneck       : None
brotli           : None
fastparquet      : 2022.11.0
fsspec           : 2022.7.1
gcsfs            : None
matplotlib       : 3.5.2
numba            : None
numexpr          : 2.8.3
odfpy            : None
openpyxl         : 3.0.10
pandas_gbq       : 0.17.7
pyarrow          : 8.0.0
pyreadstat       : None
pyxlsb           : None
s3fs             : 2022.7.1
scipy            : 1.9.0
snappy           : None
sqlalchemy       : 1.4.39
tables           : 3.7.0
tabulate         : None
xarray           : 2022.6.0
xlrd             : 2.0.1
xlwt             : None
zstandard        : None
tzdata           : None

</details>



Discussion:

Resolving patch:
diff --git a/modin/pandas/base.py b/modin/pandas/base.py
--- a/modin/pandas/base.py
+++ b/modin/pandas/base.py
@@ -245,47 +245,42 @@ def _validate_other(
         TypeError
             If any validation checks fail.
         """
-        # We skip dtype checking if the other is a scalar.
-        if is_scalar(other):
+        if isinstance(other, BasePandasDataset):
+            return other._query_compiler
+        if not is_list_like(other):
+            # We skip dtype checking if the other is a scalar. Note that pandas
+            # is_scalar can be misleading as it is False for almost all objects,
+            # even when those objects should be treated as scalars. See e.g.
+            # https://github.com/modin-project/modin/issues/5236. Therefore, we
+            # detect scalars by checking that `other` is neither a list-like nor
+            # another BasePandasDataset.
             return other
         axis = self._get_axis_number(axis) if axis is not None else 1
         result = other
-        if isinstance(other, BasePandasDataset):
-            return other._query_compiler
-        elif is_list_like(other):
-            if axis == 0:
-                if len(other) != len(self._query_compiler.index):
-                    raise ValueError(
-                        f"Unable to coerce to Series, length must be {len(self._query_compiler.index)}: "
-                        + f"given {len(other)}"
-                    )
-            else:
-                if len(other) != len(self._query_compiler.columns):
-                    raise ValueError(
-                        f"Unable to coerce to Series, length must be {len(self._query_compiler.columns)}: "
-                        + f"given {len(other)}"
-                    )
-            if hasattr(other, "dtype"):
-                other_dtypes = [other.dtype] * len(other)
-            elif is_dict_like(other):
-                other_dtypes = [
-                    type(other[label])
-                    for label in self._query_compiler.get_axis(axis)
-                    # The binary operation is applied for intersection of axis labels
-                    # and dictionary keys. So filtering out extra keys.
-                    if label in other
-                ]
-            else:
-                other_dtypes = [type(x) for x in other]
+        if axis == 0:
+            if len(other) != len(self._query_compiler.index):
+                raise ValueError(
+                    f"Unable to coerce to Series, length must be {len(self._query_compiler.index)}: "
+                    + f"given {len(other)}"
+                )
         else:
-            other_dtypes = [
-                type(other)
-                for _ in range(
-                    len(self._query_compiler.index)
-                    if axis
-                    else len(self._query_compiler.columns)
+            if len(other) != len(self._query_compiler.columns):
+                raise ValueError(
+                    f"Unable to coerce to Series, length must be {len(self._query_compiler.columns)}: "
+                    + f"given {len(other)}"
                 )
+        if hasattr(other, "dtype"):
+            other_dtypes = [other.dtype] * len(other)
+        elif is_dict_like(other):
+            other_dtypes = [
+                type(other[label])
+                for label in self._query_compiler.get_axis(axis)
+                # The binary operation is applied for intersection of axis labels
+                # and dictionary keys. So filtering out extra keys.
+                if label in other
             ]
+        else:
+            other_dtypes = [type(x) for x in other]
         if compare_index:
             if not self.index.equals(other.index):
                 raise TypeError("Cannot perform operation with non-equal index")
@@ -304,6 +299,9 @@ def _validate_other(
                     if label in other
                 ]
 
+            # TODO(https://github.com/modin-project/modin/issues/5239):
+            # this spuriously rejects other that is a list including some
+            # custom type that can be added to self's elements.
             if not all(
                 (is_numeric_dtype(self_dtype) and is_numeric_dtype(other_dtype))
                 or (is_object_dtype(self_dtype) and is_object_dtype(other_dtype))


Test patch:
diff --git a/modin/pandas/test/dataframe/test_binary.py b/modin/pandas/test/dataframe/test_binary.py
--- a/modin/pandas/test/dataframe/test_binary.py
+++ b/modin/pandas/test/dataframe/test_binary.py
@@ -16,6 +16,7 @@
 import matplotlib
 import modin.pandas as pd
 
+from modin._compat import PandasCompatVersion
 from modin.core.dataframe.pandas.partitioning.axis_partition import (
     PandasDataframeAxisPartition,
 )
@@ -27,6 +28,7 @@
     test_data,
     create_test_dfs,
     default_to_pandas_ignore_string,
+    CustomIntegerForAddition,
 )
 from modin.config import Engine, NPartitions
 from modin.test.test_utils import warns_that_defaulting_to_pandas
@@ -334,3 +336,18 @@ def test_add_string_to_df():
     modin_df, pandas_df = create_test_dfs(["a", "b"])
     eval_general(modin_df, pandas_df, lambda df: "string" + df)
     eval_general(modin_df, pandas_df, lambda df: df + "string")
+
+
+@pytest.mark.xfail(
+    PandasCompatVersion.CURRENT == PandasCompatVersion.PY36,
+    reason="Seems to be a bug in pandas 1.1.5. pandas throws ValueError "
+    + "for this particular dataframe.",
+)
+def test_add_custom_class():
+    # see https://github.com/modin-project/modin/issues/5236
+    # Test that we can add any object that is addable to pandas object data
+    # via "+".
+    eval_general(
+        *create_test_dfs(test_data["int_data"]),
+        lambda df: df + CustomIntegerForAddition(4),
+    )
diff --git a/modin/pandas/test/test_series.py b/modin/pandas/test/test_series.py
--- a/modin/pandas/test/test_series.py
+++ b/modin/pandas/test/test_series.py
@@ -74,6 +74,7 @@
     test_data_large_categorical_series_keys,
     test_data_large_categorical_series_values,
     default_to_pandas_ignore_string,
+    CustomIntegerForAddition,
 )
 from modin.config import NPartitions
 
@@ -635,6 +636,16 @@ def test_add_suffix(data):
     )
 
 
+def test_add_custom_class():
+    # see https://github.com/modin-project/modin/issues/5236
+    # Test that we can add any object that is addable to pandas object data
+    # via "+".
+    eval_general(
+        *create_test_series(test_data["int_data"]),
+        lambda df: df + CustomIntegerForAddition(4),
+    )
+
+
 @pytest.mark.parametrize("data", test_data_values, ids=test_data_keys)
 @pytest.mark.parametrize("func", agg_func_values, ids=agg_func_keys)
 def test_agg(data, func):
diff --git a/modin/pandas/test/utils.py b/modin/pandas/test/utils.py
--- a/modin/pandas/test/utils.py
+++ b/modin/pandas/test/utils.py
@@ -479,6 +479,17 @@
 time_parsing_csv_path = "modin/pandas/test/data/test_time_parsing.csv"
 
 
+class CustomIntegerForAddition:
+    def __init__(self, value: int):
+        self.value = value
+
+    def __add__(self, other):
+        return self.value + other
+
+    def __radd__(self, other):
+        return other + self.value
+
+
 def categories_equals(left, right):
     assert (left.ordered and right.ordered) or (not left.ordered and not right.ordered)
     assert_extension_array_equal(left, right)