Repository: modin-project/modin

Issue:
Enable test_io test on the HDK engine and add to ci
### Modin version checks

- [X] I have checked that this issue has not already been reported.

- [X] I have confirmed this bug exists on the latest released version of Modin.

- [X] I have confirmed this bug exists on the main branch of Modin. (In order to do this you can follow [this guide](https://modin.readthedocs.io/en/stable/getting_started/installation.html#installing-from-the-github-master-branch).)


### Reproducible Example

```python
Run test_io on the HDK engine.
```


### Issue Description

There are 48 failures, when the test is running on the HDK engine.

### Expected Behavior

All these failures should be fixed and the test added to CI.

### Error Logs

<details>

```python-traceback

Replace this line with the error backtrace (if applicable).

```

</details>


### Installed Versions

<details>

Replace this line with the output of pd.show_versions()

</details>



Discussion:

Resolving patch:
diff --git a/modin/core/io/io.py b/modin/core/io/io.py
--- a/modin/core/io/io.py
+++ b/modin/core/io/io.py
@@ -440,19 +440,21 @@ def read_sql(
         ErrorMessage.default_to_pandas("`read_sql`")
         if isinstance(con, ModinDatabaseConnection):
             con = con.get_connection()
-        return cls.from_pandas(
-            pandas.read_sql(
-                sql,
-                con,
-                index_col=index_col,
-                coerce_float=coerce_float,
-                params=params,
-                parse_dates=parse_dates,
-                columns=columns,
-                chunksize=chunksize,
-            )
+        result = pandas.read_sql(
+            sql,
+            con,
+            index_col=index_col,
+            coerce_float=coerce_float,
+            params=params,
+            parse_dates=parse_dates,
+            columns=columns,
+            chunksize=chunksize,
         )
 
+        if isinstance(result, (pandas.DataFrame, pandas.Series)):
+            return cls.from_pandas(result)
+        return (cls.from_pandas(df) for df in result)
+
     @classmethod
     @_inherit_docstrings(pandas.read_fwf, apilink="pandas.read_fwf")
     @doc(
diff --git a/modin/experimental/core/execution/native/implementations/hdk_on_native/io/io.py b/modin/experimental/core/execution/native/implementations/hdk_on_native/io/io.py
--- a/modin/experimental/core/execution/native/implementations/hdk_on_native/io/io.py
+++ b/modin/experimental/core/execution/native/implementations/hdk_on_native/io/io.py
@@ -39,6 +39,8 @@
 import pandas._libs.lib as lib
 from pandas.io.common import is_url
 
+from modin.utils import _inherit_docstrings
+
 ReadCsvKwargsType = Dict[
     str,
     Union[
@@ -508,3 +510,16 @@ def _validate_read_csv_kwargs(
 
         if on_bad_lines not in ["error", "warn", "skip", None]:
             raise ValueError(f"Argument {on_bad_lines} is invalid for on_bad_lines.")
+
+    @classmethod
+    @_inherit_docstrings(BaseIO.read_sql, apilink="pandas.read_sql")
+    def read_sql(cls, **kwargs):
+        impl = super(HdkOnNativeIO, cls)
+        varnames = impl.read_sql.__code__.co_varnames
+        filtered = {k: v for k, v in kwargs.items() if k in varnames}
+        if len(filtered) != len(kwargs):
+            if unsupported := {
+                k: v for k, v in kwargs.items() if k not in filtered and v is not None
+            }:
+                raise NotImplementedError(f"Unsupported arguments: {unsupported}")
+        return impl.read_sql(**filtered)
diff --git a/modin/experimental/pandas/io.py b/modin/experimental/pandas/io.py
--- a/modin/experimental/pandas/io.py
+++ b/modin/experimental/pandas/io.py
@@ -16,7 +16,7 @@
 import inspect
 import pathlib
 import pickle
-from typing import Union, IO, AnyStr, Callable, Optional
+from typing import Union, IO, AnyStr, Callable, Optional, Iterator
 
 import pandas
 import pandas._libs.lib as lib
@@ -24,6 +24,7 @@
 
 from . import DataFrame
 from modin.config import IsExperimental
+from modin.core.storage_formats import BaseQueryCompiler
 
 
 def read_sql(
@@ -39,7 +40,7 @@ def read_sql(
     lower_bound: Optional[int] = None,
     upper_bound: Optional[int] = None,
     max_sessions: Optional[int] = None,
-) -> DataFrame:
+) -> Union[DataFrame, Iterator[DataFrame]]:
     """
     General documentation is available in `modin.pandas.read_sql`.
 
@@ -95,7 +96,7 @@ def read_sql(
 
     Returns
     -------
-    modin.DataFrame
+    modin.DataFrame or Iterator[modin.DataFrame]
     """
     _, _, _, kwargs = inspect.getargvalues(inspect.currentframe())
 
@@ -103,7 +104,10 @@ def read_sql(
 
     assert IsExperimental.get(), "This only works in experimental mode"
 
-    return DataFrame(query_compiler=FactoryDispatcher.read_sql(**kwargs))
+    result = FactoryDispatcher.read_sql(**kwargs)
+    if isinstance(result, BaseQueryCompiler):
+        return DataFrame(query_compiler=result)
+    return (DataFrame(query_compiler=qc) for qc in result)
 
 
 def read_custom_text(


Test patch:
diff --git a/modin/pandas/test/dataframe/test_map_metadata.py b/modin/pandas/test/dataframe/test_map_metadata.py
--- a/modin/pandas/test/dataframe/test_map_metadata.py
+++ b/modin/pandas/test/dataframe/test_map_metadata.py
@@ -505,7 +505,19 @@ def test_astype_errors(errors):
     eval_general(modin_df, pandas_df, lambda df: df.astype("int", errors=errors))
 
 
-@pytest.mark.parametrize("has_dtypes", [False, True])
+@pytest.mark.parametrize(
+    "has_dtypes",
+    [
+        pytest.param(
+            False,
+            marks=pytest.mark.xfail(
+                StorageFormat.get() == "Hdk",
+                reason="HDK does not support cases when `.dtypes` is None",
+            ),
+        ),
+        True,
+    ],
+)
 def test_astype_copy(has_dtypes):
     data = [1]
     modin_df, pandas_df = pd.DataFrame(data), pandas.DataFrame(data)
diff --git a/modin/pandas/test/test_io.py b/modin/pandas/test/test_io.py
--- a/modin/pandas/test/test_io.py
+++ b/modin/pandas/test/test_io.py
@@ -1317,6 +1317,9 @@ def wrapped_read_table(file, method):
             pandas_df = wrapped_read_table(unique_filename, method="pandas")
             modin_df = wrapped_read_table(unique_filename, method="modin")
 
+        if StorageFormat.get() == "Hdk":
+            modin_df, pandas_df = align_datetime_dtypes(modin_df, pandas_df)
+
         df_equals(modin_df, pandas_df)
 
     def test_read_table_empty_frame(self, make_csv_file):
@@ -1446,6 +1449,10 @@ def test_read_parquet_partitioned_directory(
                 columns=columns,
             )
 
+    @pytest.mark.skipif(
+        StorageFormat.get() == "Hdk",
+        reason="https://github.com/intel-ai/hdk/issues/291",
+    )
     @pytest.mark.xfail(
         condition="config.getoption('--simulate-cloud').lower() != 'off'",
         reason="The reason of tests fail in `cloud` mode is unknown for now - issue #3264",
@@ -1844,7 +1851,7 @@ def test_read_excel_all_sheets(self, make_excel_file):
             df_equals(modin_df.get(key), pandas_df.get(key))
 
     @pytest.mark.xfail(
-        Engine.get() != "Python",
+        Engine.get() != "Python" and StorageFormat.get() != "Hdk",
         reason="pandas throws the exception. See pandas issue #39250 for more info",
     )
     @check_file_leaks