Repository: modin-project/modin

Issue:
Enable test_general test on the HDK engine and add to ci
### Modin version checks

- [X] I have checked that this issue has not already been reported.

- [X] I have confirmed this bug exists on the latest released version of Modin.

- [X] I have confirmed this bug exists on the main branch of Modin. (In order to do this you can follow [this guide](https://modin.readthedocs.io/en/stable/getting_started/installation.html#installing-from-the-github-master-branch).)


### Reproducible Example

```python
Run test_general on the HDK engine.
```


### Issue Description

There are 6 failures, when the test is running on the HDK engine.

### Expected Behavior

All these failures should be fixed and the test added to CI.

### Error Logs

<details>

```python-traceback

Replace this line with the error backtrace (if applicable).

```

</details>


### Installed Versions

<details>

Replace this line with the output of pd.show_versions()

</details>



Discussion:

Resolving patch:
diff --git a/modin/experimental/core/execution/native/implementations/hdk_on_native/dataframe/dataframe.py b/modin/experimental/core/execution/native/implementations/hdk_on_native/dataframe/dataframe.py
--- a/modin/experimental/core/execution/native/implementations/hdk_on_native/dataframe/dataframe.py
+++ b/modin/experimental/core/execution/native/implementations/hdk_on_native/dataframe/dataframe.py
@@ -23,6 +23,7 @@
 )
 from .utils import (
     LazyProxyCategoricalDtype,
+    arrow_to_pandas,
     check_join_supported,
     check_cols_to_join,
     get_data_for_join_by_index,
@@ -2542,7 +2543,7 @@ def to_pandas(self):
                 for idx, new_type in cast.items():
                     schema = schema.set(idx, pyarrow.field(new_type, pyarrow.string()))
                 at = at.cast(schema)
-            df = at.to_pandas()
+            df = arrow_to_pandas(at)
         else:
             df = self._partition_mgr_cls.to_pandas(self._partitions)
 
diff --git a/modin/experimental/core/execution/native/implementations/hdk_on_native/dataframe/utils.py b/modin/experimental/core/execution/native/implementations/hdk_on_native/dataframe/utils.py
--- a/modin/experimental/core/execution/native/implementations/hdk_on_native/dataframe/utils.py
+++ b/modin/experimental/core/execution/native/implementations/hdk_on_native/dataframe/utils.py
@@ -15,13 +15,16 @@
 
 from collections import OrderedDict
 
-import pandas
-import pyarrow
+import pandas as pd
+from pandas.core.arrays.arrow.extension_types import ArrowIntervalType
+
+import pyarrow as pa
+from pyarrow.types import is_dictionary
 
 from modin.error_message import ErrorMessage
 
 
-class LazyProxyCategoricalDtype(pandas.CategoricalDtype):
+class LazyProxyCategoricalDtype(pd.CategoricalDtype):
     """
     Proxy class for lazily retrieving categorical dtypes from arrow tables.
 
@@ -33,7 +36,7 @@ class LazyProxyCategoricalDtype(pandas.CategoricalDtype):
         Column name.
     """
 
-    def __init__(self, table: pyarrow.Table, column_name: str):
+    def __init__(self, table: pa.Table, column_name: str):
         ErrorMessage.catch_bugs_and_request_email(
             failure_condition=table is None,
             extra_log="attempted to bind 'None' pyarrow table to a lazy category",
@@ -43,7 +46,7 @@ def __init__(self, table: pyarrow.Table, column_name: str):
         self._ordered = False
         self._lazy_categories = None
 
-    def _new(self, table: pyarrow.Table, column_name: str) -> pandas.CategoricalDtype:
+    def _new(self, table: pa.Table, column_name: str) -> pd.CategoricalDtype:
         """
         Create a new proxy, if either table or column name are different.
 
@@ -60,7 +63,7 @@ def _new(self, table: pyarrow.Table, column_name: str) -> pandas.CategoricalDtyp
         """
         if self._table is None:
             # The table has been materialized, we don't need a proxy anymore.
-            return pandas.CategoricalDtype(self.categories)
+            return pd.CategoricalDtype(self.categories)
         elif table is self._table and column_name == self._column_name:
             return self
         else:
@@ -70,7 +73,7 @@ def _new(self, table: pyarrow.Table, column_name: str) -> pandas.CategoricalDtyp
     def _categories(self):  # noqa: GL08
         if self._table is not None:
             chunks = self._table.column(self._column_name).chunks
-            cat = pandas.concat([chunk.dictionary.to_pandas() for chunk in chunks])
+            cat = pd.concat([chunk.dictionary.to_pandas() for chunk in chunks])
             self._lazy_categories = self.validate_categories(cat.unique())
             self._table = None  # The table is not required any more
         return self._lazy_categories
@@ -193,10 +196,10 @@ def to_empty_pandas_df(df):
             if len(df._index_cols) > 1:
                 arrays = [[i] for i in range(len(df._index_cols))]
                 names = [df._index_name(n) for n in df._index_cols]
-                idx = pandas.MultiIndex.from_arrays(arrays, names=names)
+                idx = pd.MultiIndex.from_arrays(arrays, names=names)
             else:
-                idx = pandas.Index(name=df._index_name(df._index_cols[0]))
-        return pandas.DataFrame(columns=df.columns, index=idx)
+                idx = pd.Index(name=df._index_name(df._index_cols[0]))
+        return pd.DataFrame(columns=df.columns, index=idx)
 
     new_dtypes = []
     exprs = OrderedDict()
@@ -248,3 +251,44 @@ def to_empty_pandas_df(df):
         new_dtypes.append(df._dtypes[orig_name])
 
     return index_cols, exprs, new_dtypes, merged.columns
+
+
+def arrow_to_pandas(at: pa.Table) -> pd.DataFrame:
+    """
+    Convert the specified arrow table to pandas.
+
+    Parameters
+    ----------
+    at : pyarrow.Table
+        The table to convert.
+
+    Returns
+    -------
+    pandas.DataFrame
+    """
+
+    def mapper(at):
+        if is_dictionary(at) and isinstance(at.value_type, ArrowIntervalType):
+            # The default mapper fails with TypeError: unhashable type: 'dict'
+            return _CategoricalDtypeMapper
+        return None
+
+    return at.to_pandas(types_mapper=mapper)
+
+
+class _CategoricalDtypeMapper:  # noqa: GL08
+    @staticmethod
+    def __from_arrow__(arr):  # noqa: GL08
+        values = []
+        # Using OrderedDict as an ordered set to preserve the categories order
+        categories = OrderedDict()
+        chunks = arr.chunks if isinstance(arr, pa.ChunkedArray) else (arr,)
+        for chunk in chunks:
+            assert isinstance(chunk, pa.DictionaryArray)
+            cat = chunk.dictionary.to_pandas()
+            values.append(chunk.indices.to_pandas().map(cat))
+            categories.update((c, None) for c in cat)
+        return pd.Categorical(
+            pd.concat(values, ignore_index=True),
+            dtype=pd.CategoricalDtype(categories, ordered=True),
+        )
diff --git a/modin/experimental/core/execution/native/implementations/hdk_on_native/partitioning/partition.py b/modin/experimental/core/execution/native/implementations/hdk_on_native/partitioning/partition.py
--- a/modin/experimental/core/execution/native/implementations/hdk_on_native/partitioning/partition.py
+++ b/modin/experimental/core/execution/native/implementations/hdk_on_native/partitioning/partition.py
@@ -16,6 +16,7 @@
 import pandas
 
 from modin.core.dataframe.pandas.partitioning.partition import PandasDataframePartition
+from ..dataframe.utils import arrow_to_pandas
 from ..db_worker import DbWorker
 import pyarrow
 
@@ -82,7 +83,7 @@ def to_pandas(self):
         if isinstance(obj, (pandas.DataFrame, pandas.Series)):
             return obj
         assert isinstance(obj, pyarrow.Table)
-        return obj.to_pandas()
+        return arrow_to_pandas(obj)
 
     def get(self):
         """


Test patch:
diff --git a/modin/pandas/test/test_general.py b/modin/pandas/test/test_general.py
--- a/modin/pandas/test/test_general.py
+++ b/modin/pandas/test/test_general.py
@@ -19,7 +19,7 @@
 from numpy.testing import assert_array_equal
 from modin.utils import get_current_execution, to_pandas
 from modin.test.test_utils import warns_that_defaulting_to_pandas
-from modin.config import Engine
+from modin.config import Engine, StorageFormat
 from pandas.testing import assert_frame_equal
 
 from .utils import (
@@ -31,9 +31,14 @@
     eval_general,
     bool_arg_values,
     bool_arg_keys,
+    default_to_pandas_ignore_string,
 )
 
 
+if StorageFormat.get() == "Hdk":
+    pytestmark = pytest.mark.filterwarnings(default_to_pandas_ignore_string)
+
+
 @contextlib.contextmanager
 def _nullcontext():
     """Replacement for contextlib.nullcontext missing in older Python."""
@@ -635,6 +640,10 @@ def test_unique():
     assert modin_result.shape == pandas_result.shape
 
 
+@pytest.mark.xfail(
+    StorageFormat.get() == "Hdk",
+    reason="https://github.com/modin-project/modin/issues/2896",
+)
 @pytest.mark.parametrize("normalize, bins, dropna", [(True, 3, False)])
 def test_value_counts(normalize, bins, dropna):
     # We sort indices for Modin and pandas result because of issue #1650
@@ -770,7 +779,9 @@ def test_to_pandas_indices(data):
         assert md_df.axes[axis].equals(
             pd_df.axes[axis]
         ), f"Indices at axis {axis} are different!"
-        assert md_df.axes[axis].equal_levels(
+        assert not hasattr(md_df.axes[axis], "equal_levels") or md_df.axes[
+            axis
+        ].equal_levels(
             pd_df.axes[axis]
         ), f"Levels of indices at axis {axis} are different!"
 
@@ -829,7 +840,7 @@ def test_default_to_pandas_warning_message(func, regex):
 
 def test_empty_dataframe():
     df = pd.DataFrame(columns=["a", "b"])
-    with warns_that_defaulting_to_pandas():
+    with warns_that_defaulting_to_pandas() if StorageFormat.get() != "Hdk" else _nullcontext():
         df[(df.a == 1) & (df.b == 2)]