Repository: modin-project/modin

Issue:
Enable test_series test on the HDK engine and add to ci
### Modin version checks

- [X] I have checked that this issue has not already been reported.

- [X] I have confirmed this bug exists on the latest released version of Modin.

- [X] I have confirmed this bug exists on the main branch of Modin. (In order to do this you can follow [this guide](https://modin.readthedocs.io/en/stable/getting_started/installation.html#installing-from-the-github-master-branch).)


### Reproducible Example

```python
Run test_series on the HDK engine.
```


### Issue Description

There are 92 failures, when the test is running on the HDK engine.

### Expected Behavior

All these failures should be fixed and the test added to CI.

### Error Logs

<details>

```python-traceback

Replace this line with the error backtrace (if applicable).

```

</details>


### Installed Versions

<details>

Replace this line with the output of pd.show_versions()

</details>



Discussion:

Resolving patch:
diff --git a/modin/experimental/core/execution/native/implementations/hdk_on_native/dataframe/dataframe.py b/modin/experimental/core/execution/native/implementations/hdk_on_native/dataframe/dataframe.py
--- a/modin/experimental/core/execution/native/implementations/hdk_on_native/dataframe/dataframe.py
+++ b/modin/experimental/core/execution/native/implementations/hdk_on_native/dataframe/dataframe.py
@@ -515,7 +515,10 @@ def generate_by_name(by):
 
         # TODO: check performance changes after enabling 'dropna' and decide
         # is it worth it or not.
-        # if groupby_args["dropna"]:
+        if groupby_args["dropna"]:
+            ErrorMessage.single_warning(
+                "'dropna' is temporary disabled due to https://github.com/modin-project/modin/issues/2896"
+            )
         #     base = base.dropna(subset=groupby_cols, how="any")
 
         if as_index:
@@ -1564,18 +1567,18 @@ def cat_codes(self):
         assert len(self.columns) == 1
         assert self._dtypes[-1] == "category"
 
-        col = self.columns[-1]
         exprs = self._index_exprs()
-        col_expr = self.ref(col)
+        col_expr = self.ref(self.columns[-1])
         code_expr = OpExpr("KEY_FOR_STRING", [col_expr], get_dtype("int32"))
         null_val = LiteralExpr(np.int32(-1))
-        exprs[col] = build_if_then_else(
+        col_name = MODIN_UNNAMED_SERIES_LABEL
+        exprs[col_name] = build_if_then_else(
             col_expr.is_null(), null_val, code_expr, get_dtype("int32")
         )
 
         return self.__constructor__(
-            columns=self.columns,
-            dtypes=self._dtypes,
+            columns=Index([col_name]),
+            dtypes=pd.Series(self._dtypes[0], index=Index([col_name])),
             op=TransformNode(self, exprs),
             index=self._index_cache,
             index_cols=self._index_cols,
diff --git a/modin/experimental/core/storage_formats/hdk/query_compiler.py b/modin/experimental/core/storage_formats/hdk/query_compiler.py
--- a/modin/experimental/core/storage_formats/hdk/query_compiler.py
+++ b/modin/experimental/core/storage_formats/hdk/query_compiler.py
@@ -29,7 +29,7 @@
 import pandas
 from pandas._libs.lib import no_default
 from pandas.core.common import is_bool_indexer
-from pandas.core.dtypes.common import is_list_like
+from pandas.core.dtypes.common import is_list_like, is_bool_dtype, is_integer_dtype
 from functools import wraps
 
 import numpy as np
@@ -528,6 +528,14 @@ def fillna(
         downcast=None,
     ):
         assert not inplace, "inplace=True should be handled on upper level"
+
+        if (
+            isinstance(value, dict)
+            and len(self._modin_frame.columns) == 1
+            and self._modin_frame.columns[0] == MODIN_UNNAMED_SERIES_LABEL
+        ):
+            raise NotImplementedError("Series fillna with dict value")
+
         new_frame = self._modin_frame.fillna(
             value=value,
             method=method,
@@ -657,11 +665,19 @@ def mul(self, other, **kwargs):
         return self._bin_op(other, "mul", **kwargs)
 
     def mod(self, other, **kwargs):
-        int_codes = np.typecodes["AllInteger"]
-        if all(t.char in int_codes for t in self._modin_frame.dtypes):
-            return self._bin_op(other, "mod", **kwargs)
-        else:
-            raise NotImplementedError("Non-integer operands in modulo operation")
+        def check_int(obj):
+            if isinstance(obj, DFAlgQueryCompiler):
+                cond = all(is_integer_dtype(t) for t in obj._modin_frame.dtypes)
+            elif isinstance(obj, list):
+                cond = all(isinstance(i, int) for i in obj)
+            else:
+                cond = isinstance(obj, int)
+            if not cond:
+                raise NotImplementedError("Non-integer operands in modulo operation")
+
+        check_int(self)
+        check_int(other)
+        return self._bin_op(other, "mod", **kwargs)
 
     def floordiv(self, other, **kwargs):
         return self._bin_op(other, "floordiv", **kwargs)
@@ -688,10 +704,25 @@ def ne(self, other, **kwargs):
         return self._bin_op(other, "ne", **kwargs)
 
     def __and__(self, other, **kwargs):
-        return self._bin_op(other, "and", **kwargs)
+        return self._bool_op(other, "and", **kwargs)
 
     def __or__(self, other, **kwargs):
-        return self._bin_op(other, "or", **kwargs)
+        return self._bool_op(other, "or", **kwargs)
+
+    def _bool_op(self, other, op, **kwargs):  # noqa: GL08
+        def check_bool(obj):
+            if isinstance(obj, DFAlgQueryCompiler):
+                cond = all(is_bool_dtype(t) for t in obj._modin_frame.dtypes)
+            elif isinstance(obj, list):
+                cond = all(isinstance(i, bool) for i in obj)
+            else:
+                cond = isinstance(obj, bool)
+            if not cond:
+                raise NotImplementedError("Non-boolean operands in logic operation")
+
+        check_bool(self)
+        check_bool(other)
+        return self._bin_op(other, op, **kwargs)
 
     def reset_index(self, **kwargs):
         level = kwargs.get("level", None)


Test patch:
diff --git a/modin/pandas/test/test_series.py b/modin/pandas/test/test_series.py
--- a/modin/pandas/test/test_series.py
+++ b/modin/pandas/test/test_series.py
@@ -15,6 +15,7 @@
 import numpy as np
 import json
 import pandas
+from pandas._testing import assert_series_equal
 from pandas.errors import SpecificationError
 from pandas.core.indexing import IndexingError
 import matplotlib
@@ -77,7 +78,7 @@
     CustomIntegerForAddition,
     NonCommutativeMultiplyInteger,
 )
-from modin.config import NPartitions
+from modin.config import NPartitions, StorageFormat
 
 # Our configuration in pytest.ini requires that we explicitly catch all
 # instances of defaulting to pandas, but some test modules, like this one,
@@ -91,6 +92,9 @@
 # Force matplotlib to not use any Xwindows backend.
 matplotlib.use("Agg")
 
+# Initialize the environment
+pd.DataFrame()
+
 
 def get_rop(op):
     if op.startswith("__") and op.endswith("__"):
@@ -596,6 +600,10 @@ def test___str__(data):
     assert str(modin_series) == str(pandas_series)
 
 
+@pytest.mark.skipif(
+    StorageFormat.get() == "Hdk",
+    reason="https://github.com/intel-ai/hdk/issues/272",
+)
 @pytest.mark.parametrize("data", test_data_values, ids=test_data_keys)
 def test___sub__(data):
     modin_series, pandas_series = create_test_series(data)
@@ -1047,6 +1055,10 @@ def test_asof_large(where):
 
 
 @pytest.mark.parametrize("data", test_data_values, ids=test_data_keys)
+@pytest.mark.xfail(
+    StorageFormat.get() == "Hdk",
+    reason="https://github.com/intel-ai/hdk/issues/186",
+)
 def test_astype(data):
     modin_series, pandas_series = create_test_series(data)
     series_name = "test_series"
@@ -1255,6 +1267,10 @@ def test_clip_sequence(request, data, bound_type):
         df_equals(modin_result, pandas_result)
 
 
+@pytest.mark.skipif(
+    StorageFormat.get() == "Hdk",
+    reason="https://github.com/intel-ai/hdk/issues/271",
+)
 @pytest.mark.parametrize("data", test_data_values, ids=test_data_keys)
 def test_combine(data):
     modin_series, _ = create_test_series(data)  # noqa: F841
@@ -1262,6 +1278,10 @@ def test_combine(data):
     modin_series.combine(modin_series2, lambda s1, s2: s1 if s1 < s2 else s2)
 
 
+@pytest.mark.skipif(
+    StorageFormat.get() == "Hdk",
+    reason="https://github.com/intel-ai/hdk/issues/271",
+)
 @pytest.mark.parametrize("data", test_data_values, ids=test_data_keys)
 def test_combine_first(data):
     modin_series, pandas_series = create_test_series(data)
@@ -1657,7 +1677,19 @@ def test_dtype(data):
 # the frame is created has timezone None, so that its dtype is datetime64[ns]
 # as opposed to, e.g. datetime64[ns, Europe/Berlin]. To reproduce that bug, we
 # use timezones None and Europe/Berlin.
-@pytest.mark.parametrize("timezone", [None, "Europe/Berlin"])
+@pytest.mark.parametrize(
+    "timezone",
+    [
+        pytest.param(None),
+        pytest.param(
+            "Europe/Berlin",
+            marks=pytest.mark.skipif(
+                StorageFormat.get() == "Hdk",
+                reason="HDK is unable to store TZ in the table schema",
+            ),
+        ),
+    ],
+)
 def test_dt(timezone):
     data = pd.date_range("2016-12-31", periods=128, freq="D", tz=timezone)
     modin_series = pd.Series(data)
@@ -1747,11 +1779,15 @@ def dt_with_empty_partition(lib):
         df_b = lib.DataFrame({"B": [lib.to_datetime("27/10/2020")]})
         df = lib.concat([df_a, df_b], axis=1)
         eval_result = df.eval("B - A", engine="python")
-        # BaseOnPython had a single partition after the concat, and it
+        # BaseOnPython ahd HDK had a single partition after the concat, and it
         # maintains that partition after eval. In other execution modes,
         # eval() should re-split the result into two column partitions,
         # one of which is empty.
-        if isinstance(df, pd.DataFrame) and get_current_execution() != "BaseOnPython":
+        if (
+            isinstance(df, pd.DataFrame)
+            and get_current_execution() != "BaseOnPython"
+            and StorageFormat.get() != "Hdk"
+        ):
             assert eval_result._query_compiler._modin_frame._partitions.shape == (1, 2)
         return eval_result.dt.days
 
@@ -3226,12 +3262,20 @@ def test_std(request, data, skipna, ddof):
         df_equals(modin_result, pandas_result)
 
 
+@pytest.mark.skipif(
+    StorageFormat.get() == "Hdk",
+    reason="https://github.com/intel-ai/hdk/issues/272",
+)
 @pytest.mark.parametrize("data", test_data_values, ids=test_data_keys)
 def test_sub(data):
     modin_series, pandas_series = create_test_series(data)
     inter_df_math_helper(modin_series, pandas_series, "sub")
 
 
+@pytest.mark.skipif(
+    StorageFormat.get() == "Hdk",
+    reason="https://github.com/intel-ai/hdk/issues/272",
+)
 @pytest.mark.parametrize("data", test_data_values, ids=test_data_keys)
 def test_subtract(data):
     modin_series, pandas_series = create_test_series(data)
@@ -3584,7 +3628,20 @@ def test_update(data, other_data):
 @pytest.mark.parametrize("sort", bool_arg_values, ids=bool_arg_keys)
 @pytest.mark.parametrize("normalize", bool_arg_values, ids=bool_arg_keys)
 @pytest.mark.parametrize("bins", [3, None])
-@pytest.mark.parametrize("dropna", bool_arg_values, ids=bool_arg_keys)
+@pytest.mark.parametrize(
+    "dropna",
+    [
+        pytest.param(None),
+        pytest.param(False),
+        pytest.param(
+            True,
+            marks=pytest.mark.skipif(
+                StorageFormat.get() == "Hdk",
+                reason="https://github.com/modin-project/modin/issues/2896",
+            ),
+        ),
+    ],
+)
 @pytest.mark.parametrize("ascending", bool_arg_values, ids=bool_arg_keys)
 def test_value_counts(sort, normalize, bins, dropna, ascending):
     def sort_sensitive_comparator(df1, df2):
@@ -3637,9 +3694,19 @@ def test_value_counts_categorical():
     random_state = np.random.RandomState(seed=42)
     random_state.shuffle(data)
 
+    if StorageFormat.get() == "Hdk":
+        # The order of HDK categories is different from Pandas
+        # and, thus, index comparison fails.
+        def comparator(df1, df2):
+            assert_series_equal(df1._to_pandas(), df2, check_index=False)
+
+    else:
+        comparator = df_equals
+
     eval_general(
         *create_test_series(data, dtype="category"),
         lambda df: df.value_counts(),
+        comparator=comparator,
     )
 
 
@@ -4383,6 +4450,10 @@ def test_cat_ordered(data):
     assert modin_series.cat.ordered == pandas_series.cat.ordered
 
 
+@pytest.mark.skipif(
+    StorageFormat.get() == "Hdk",
+    reason="HDK uses internal codes, that are different from Pandas",
+)
 @pytest.mark.parametrize(
     "data", test_data_categorical_values, ids=test_data_categorical_keys
 )