Repository: modin-project/modin

Issue:
Enable test_default test on the HDK engine and add to ci
### Modin version checks

- [X] I have checked that this issue has not already been reported.

- [X] I have confirmed this bug exists on the latest released version of Modin.

- [X] I have confirmed this bug exists on the main branch of Modin. (In order to do this you can follow [this guide](https://modin.readthedocs.io/en/stable/getting_started/installation.html#installing-from-the-github-master-branch).)


### Reproducible Example

```python
Run test_default on the HDK engine.
```


### Issue Description

There are 141 failures, when the test is running on the HDK engine.

### Expected Behavior

All these failures should be fixed and the test added to CI.

### Error Logs

<details>

```python-traceback

Replace this line with the error backtrace (if applicable).

```

</details>


### Installed Versions

<details>

Replace this line with the output of pd.show_versions()

</details>



Discussion:

Resolving patch:
diff --git a/modin/experimental/core/execution/native/implementations/hdk_on_native/dataframe/dataframe.py b/modin/experimental/core/execution/native/implementations/hdk_on_native/dataframe/dataframe.py
--- a/modin/experimental/core/execution/native/implementations/hdk_on_native/dataframe/dataframe.py
+++ b/modin/experimental/core/execution/native/implementations/hdk_on_native/dataframe/dataframe.py
@@ -28,6 +28,7 @@
     check_join_supported,
     check_cols_to_join,
     get_data_for_join_by_index,
+    get_common_arrow_type,
 )
 from ..partitioning.partition_manager import HdkOnNativeDataframePartitionManager
 
@@ -1124,21 +1125,14 @@ def __init__(self, frame):
         frames: List[FrameData] = [FrameData(f) for f in [self] + other_modin_frames]
         col_fields: typing.OrderedDict[Tuple[str, str], pyarrow.Field] = OrderedDict()
 
+        # Add field to the col_fields dictionary. If the field is already exists, chose
+        # the most appropriate field, according to the fields type and bit_width.
         def add_col_field(table, col_name, table_col_name):
             key = (col_name, table_col_name)
             field = table.field(table_col_name)
             cur_field = col_fields.get(key, None)
-            if (
-                (cur_field is None)
-                or pyarrow.types.is_null(cur_field.type)
-                or (
-                    (not pyarrow.types.is_string(cur_field.type))
-                    and (not pyarrow.types.is_null(field.type))
-                    and (
-                        pyarrow.types.is_string(field.type)
-                        or (field.type.bit_width > cur_field.type.bit_width)
-                    )
-                )
+            if cur_field is None or (
+                cur_field.type != get_common_arrow_type(cur_field.type, field.type)
             ):
                 col_fields[key] = field
 
@@ -1235,7 +1229,11 @@ def add_col_field(table, col_name, table_col_name):
                     idx = frames[0].index.append([f.index for f in frames[1:]])
                     idx_cols = self._mangle_index_names(idx.names)
                     idx_df = pd.DataFrame(index=idx).reset_index()
-                    idx_table = pyarrow.Table.from_pandas(idx_df)
+                    obj_cols = idx_df.select_dtypes(include=["object"]).columns.tolist()
+                    if len(obj_cols) != 0:
+                        # PyArrow fails to convert object fields. Converting to str.
+                        idx_df[obj_cols] = idx_df[obj_cols].astype(str)
+                    idx_table = pyarrow.Table.from_pandas(idx_df, preserve_index=False)
                     idx_table = idx_table.rename_columns(idx_cols)
 
             if sort:
diff --git a/modin/experimental/core/execution/native/implementations/hdk_on_native/dataframe/utils.py b/modin/experimental/core/execution/native/implementations/hdk_on_native/dataframe/utils.py
--- a/modin/experimental/core/execution/native/implementations/hdk_on_native/dataframe/utils.py
+++ b/modin/experimental/core/execution/native/implementations/hdk_on_native/dataframe/utils.py
@@ -379,6 +379,34 @@ def to_empty_pandas_df(df):
     return index_cols, exprs, new_dtypes, merged.columns
 
 
+def get_common_arrow_type(t1: pa.lib.DataType, t2: pa.lib.DataType) -> pa.lib.DataType:
+    """
+    Get common arrow data type.
+
+    Parameters
+    ----------
+    t1 : pa.lib.DataType
+    t2 : pa.lib.DataType
+
+    Returns
+    -------
+    pa.lib.DataType
+    """
+    if pa.types.is_string(t1):
+        return t1
+    if pa.types.is_string(t2):
+        return t2
+    if pa.types.is_null(t1):
+        return t2
+    if pa.types.is_null(t2):
+        return t1
+    if t1.bit_width > t2.bit_width:
+        return t1
+    if t1.bit_width < t2.bit_width:
+        return t2
+    return t2 if pa.types.is_floating(t2) else t1
+
+
 def arrow_to_pandas(at: pa.Table) -> pd.DataFrame:
     """
     Convert the specified arrow table to pandas.
diff --git a/modin/experimental/core/execution/native/implementations/hdk_on_native/partitioning/partition.py b/modin/experimental/core/execution/native/implementations/hdk_on_native/partitioning/partition.py
--- a/modin/experimental/core/execution/native/implementations/hdk_on_native/partitioning/partition.py
+++ b/modin/experimental/core/execution/native/implementations/hdk_on_native/partitioning/partition.py
@@ -85,6 +85,21 @@ def to_pandas(self):
         assert isinstance(obj, pyarrow.Table)
         return arrow_to_pandas(obj)
 
+    def to_numpy(self, **kwargs):
+        """
+        Transform to NumPy format.
+
+        Parameters
+        ----------
+        **kwargs : dict
+            Additional keyword arguments to be passed in ``to_numpy``.
+
+        Returns
+        -------
+        np.ndarray
+        """
+        return self.to_pandas().to_numpy(**kwargs)
+
     def get(self):
         """
         Get partition data.


Test patch:
diff --git a/modin/pandas/test/dataframe/test_default.py b/modin/pandas/test/dataframe/test_default.py
--- a/modin/pandas/test/dataframe/test_default.py
+++ b/modin/pandas/test/dataframe/test_default.py
@@ -45,7 +45,7 @@
     test_data_large_categorical_dataframe,
     default_to_pandas_ignore_string,
 )
-from modin.config import NPartitions
+from modin.config import NPartitions, StorageFormat
 from modin.test.test_utils import warns_that_defaulting_to_pandas
 
 NPartitions.put(4)
@@ -242,16 +242,28 @@ def test_corr(min_periods):
 @pytest.mark.parametrize("min_periods", [1, 3, 5])
 @pytest.mark.parametrize("ddof", [1, 2, 4])
 def test_cov(min_periods, ddof):
+    # Modin result may slightly differ from pandas result
+    # due to floating pointing arithmetic.
+    if StorageFormat.get() == "Hdk":
+
+        def comparator1(df1, df2):
+            modin_df_almost_equals_pandas(df1, df2, max_diff=0.0002)
+
+        comparator2 = comparator1
+    else:
+        comparator1 = df_equals
+        comparator2 = modin_df_almost_equals_pandas
+
     eval_general(
         *create_test_dfs(test_data["int_data"]),
         lambda df: df.cov(min_periods=min_periods, ddof=ddof),
+        comparator=comparator1,
     )
-    # Modin result may slightly differ from pandas result
-    # due to floating pointing arithmetic.
+
     eval_general(
         *create_test_dfs(test_data["float_nan_data"]),
         lambda df: df.cov(min_periods=min_periods),
-        comparator=modin_df_almost_equals_pandas,
+        comparator=comparator2,
     )
 
 
@@ -472,11 +484,21 @@ def test_mad_level(level):
     "value_vars", [lambda df: df.columns[-1], lambda df: df.columns[-4:], None]
 )
 def test_melt(data, id_vars, value_vars):
+    if StorageFormat.get() == "Hdk":
+        # Drop NA and sort by all columns to make sure the order
+        # is identical to Pandas.
+        def melt(df, *args, **kwargs):
+            df = df.melt(*args, **kwargs).dropna()
+            return df.sort_values(df.columns.tolist())
+
+    else:
+
+        def melt(df, *args, **kwargs):
+            return df.melt(*args, **kwargs).sort_values(["variable", "value"])
+
     eval_general(
         *create_test_dfs(data),
-        lambda df, *args, **kwargs: df.melt(*args, **kwargs)
-        .sort_values(["variable", "value"])
-        .reset_index(drop=True),
+        lambda df, *args, **kwargs: melt(df, *args, **kwargs).reset_index(drop=True),
         id_vars=id_vars,
         value_vars=value_vars,
     )
@@ -1186,12 +1208,13 @@ def test_setattr_axes():
             # In BaseOnPython, setting columns raises a warning because get_axis
             #  defaults to pandas.
             warnings.simplefilter("error")
-        df.index = ["foo", "bar"]
-        df.columns = [9, 10]
+        if StorageFormat.get() != "Hdk":  # Not yet supported - #1766
+            df.index = ["foo", "bar"]
+            # Check that ensure_index was called
+            pandas.testing.assert_index_equal(df.index, pandas.Index(["foo", "bar"]))
 
-    # Check that ensure_index was called
-    pandas.testing.assert_index_equal(df.index, pandas.Index(["foo", "bar"]))
-    pandas.testing.assert_index_equal(df.columns, pandas.Index([9, 10]))
+        df.columns = [9, 10]
+        pandas.testing.assert_index_equal(df.columns, pandas.Index([9, 10]))
 
 
 @pytest.mark.parametrize("data", test_data_values, ids=test_data_keys)
diff --git a/modin/pandas/test/utils.py b/modin/pandas/test/utils.py
--- a/modin/pandas/test/utils.py
+++ b/modin/pandas/test/utils.py
@@ -669,7 +669,7 @@ def df_equals(df1, df2):
             np.testing.assert_almost_equal(df1, df2)
 
 
-def modin_df_almost_equals_pandas(modin_df, pandas_df):
+def modin_df_almost_equals_pandas(modin_df, pandas_df, max_diff=0.0001):
     df_categories_equals(modin_df._to_pandas(), pandas_df)
 
     modin_df = to_pandas(modin_df)
@@ -679,15 +679,22 @@ def modin_df_almost_equals_pandas(modin_df, pandas_df):
     if hasattr(pandas_df, "select_dtypes"):
         pandas_df = pandas_df.select_dtypes(exclude=["category"])
 
-    difference = modin_df - pandas_df
-    diff_max = difference.max()
-    if isinstance(diff_max, pandas.Series):
-        diff_max = diff_max.max()
-    assert (
-        modin_df.equals(pandas_df)
-        or diff_max < 0.0001
-        or (all(modin_df.isna().all()) and all(pandas_df.isna().all()))
-    )
+    if modin_df.equals(pandas_df):
+        return
+
+    isna = modin_df.isna().all()
+    if isinstance(isna, bool):
+        if isna:
+            assert pandas_df.isna().all()
+            return
+    elif isna.all():
+        assert pandas_df.isna().all().all()
+        return
+
+    diff = (modin_df - pandas_df).abs()
+    diff /= pandas_df
+    diff_max = diff.max() if isinstance(diff, pandas.Series) else diff.max().max()
+    assert diff_max < max_diff, f"{diff_max} >= {max_diff}"
 
 
 def try_modin_df_almost_equals_compare(df1, df2):