Repository: modin-project/modin

Issue:
Fix HDK tests in `...hdk/test_protocol.py`
All tests that xfailed with the following reason: `conversion from 'pyarrow' ends up with the wrong datetime64 resolution`.


Discussion:

Resolving patch:
diff --git a/modin/experimental/core/execution/native/implementations/hdk_on_native/dataframe/utils.py b/modin/experimental/core/execution/native/implementations/hdk_on_native/dataframe/utils.py
--- a/modin/experimental/core/execution/native/implementations/hdk_on_native/dataframe/utils.py
+++ b/modin/experimental/core/execution/native/implementations/hdk_on_native/dataframe/utils.py
@@ -513,7 +513,15 @@ def mapper(at):
             return _CategoricalDtypeMapper
         return None
 
-    return at.to_pandas(types_mapper=mapper)
+    df = at.to_pandas(types_mapper=mapper)
+    dtype = {}
+    for idx, _type in enumerate(at.schema.types):
+        if isinstance(_type, pa.lib.TimestampType) and _type.unit != "ns":
+            dtype[at.schema.names[idx]] = f"datetime64[{_type.unit}]"
+    if dtype:
+        # TODO: remove after https://github.com/apache/arrow/pull/35656 is merge
+        df = df.astype(dtype)
+    return df
 
 
 class _CategoricalDtypeMapper:  # noqa: GL08


Test patch:
diff --git a/modin/test/interchange/dataframe_protocol/hdk/test_protocol.py b/modin/test/interchange/dataframe_protocol/hdk/test_protocol.py
--- a/modin/test/interchange/dataframe_protocol/hdk/test_protocol.py
+++ b/modin/test/interchange/dataframe_protocol/hdk/test_protocol.py
@@ -30,22 +30,10 @@
 from .utils import get_data_of_all_types, split_df_into_chunks, export_frame
 
 
-@pytest.mark.parametrize(
-    "exclude_datetime",
-    [
-        True,
-        pytest.param(
-            False,
-            marks=pytest.mark.xfail(
-                reason="conversion from 'pyarrow' ends up with the wrong datetime64 resolution"
-            ),
-        ),
-    ],
-)
 @pytest.mark.parametrize("data_has_nulls", [True, False])
 @pytest.mark.parametrize("from_hdk", [True, False])
 @pytest.mark.parametrize("n_chunks", [None, 3, 5, 12])
-def test_simple_export(data_has_nulls, from_hdk, n_chunks, exclude_datetime):
+def test_simple_export(data_has_nulls, from_hdk, n_chunks):
     if from_hdk:
         # HDK can't import 'uint64' as well as booleans
         # issue for bool: https://github.com/modin-project/modin/issues/4299
@@ -53,9 +41,6 @@ def test_simple_export(data_has_nulls, from_hdk, n_chunks, exclude_datetime):
     else:
         exclude_dtypes = []
 
-    if exclude_datetime:
-        exclude_dtypes += ["datetime"]
-
     data = get_data_of_all_types(
         has_nulls=data_has_nulls, exclude_dtypes=exclude_dtypes
     )
@@ -65,29 +50,12 @@ def test_simple_export(data_has_nulls, from_hdk, n_chunks, exclude_datetime):
     df_equals(md_df, exported_df)
 
 
-@pytest.mark.parametrize(
-    "exclude_datetime",
-    [
-        True,
-        pytest.param(
-            False,
-            marks=pytest.mark.xfail(
-                reason="conversion from 'pyarrow' ends up with the wrong datetime64 resolution"
-            ),
-        ),
-    ],
-)
 @pytest.mark.parametrize("n_chunks", [2, 4, 7])
 @pytest.mark.parametrize("data_has_nulls", [True, False])
-def test_export_aligned_at_chunks(n_chunks, data_has_nulls, exclude_datetime):
+def test_export_aligned_at_chunks(n_chunks, data_has_nulls):
     """Test export from DataFrame exchange protocol when internal PyArrow table is equaly chunked."""
-    exclude_dtypes = ["category"]
-    if exclude_datetime:
-        exclude_dtypes += ["datetime"]
     # Modin DataFrame constructor can't process PyArrow's category when using `from_arrow`, so exclude it
-    data = get_data_of_all_types(
-        has_nulls=data_has_nulls, exclude_dtypes=exclude_dtypes
-    )
+    data = get_data_of_all_types(has_nulls=data_has_nulls, exclude_dtypes=["category"])
     pd_df = pandas.DataFrame(data)
     pd_chunks = split_df_into_chunks(pd_df, n_chunks)
 
@@ -112,20 +80,8 @@ def test_export_aligned_at_chunks(n_chunks, data_has_nulls, exclude_datetime):
     df_equals(md_df, exported_df)
 
 
-@pytest.mark.parametrize(
-    "exclude_datetime",
-    [
-        True,
-        pytest.param(
-            False,
-            marks=pytest.mark.xfail(
-                reason="conversion from 'pyarrow' ends up with the wrong datetime64 resolution"
-            ),
-        ),
-    ],
-)
 @pytest.mark.parametrize("data_has_nulls", [True, False])
-def test_export_unaligned_at_chunks(data_has_nulls, exclude_datetime):
+def test_export_unaligned_at_chunks(data_has_nulls):
     """
     Test export from DataFrame exchange protocol when internal PyArrow table's chunks are unaligned.
 
@@ -133,13 +89,8 @@ def test_export_unaligned_at_chunks(data_has_nulls, exclude_datetime):
     each column has its individual chunking and so some preprocessing is required in order
     to emulate equaly chunked columns in the protocol.
     """
-    exclude_dtypes = ["category"]
-    if exclude_datetime:
-        exclude_dtypes += ["datetime"]
     # Modin DataFrame constructor can't process PyArrow's category when using `from_arrow`, so exclude it
-    data = get_data_of_all_types(
-        has_nulls=data_has_nulls, exclude_dtypes=exclude_dtypes
-    )
+    data = get_data_of_all_types(has_nulls=data_has_nulls, exclude_dtypes=["category"])
     pd_df = pandas.DataFrame(data)
     # divide columns in 3 groups: unchunked, 2-chunked, 7-chunked
     chunk_groups = [1, 2, 7]
@@ -188,32 +139,15 @@ def test_export_unaligned_at_chunks(data_has_nulls, exclude_datetime):
     df_equals(md_df, exported_df)
 
 
-@pytest.mark.parametrize(
-    "exclude_datetime",
-    [
-        True,
-        pytest.param(
-            False,
-            marks=pytest.mark.xfail(
-                reason="conversion from 'pyarrow' ends up with the wrong datetime64 resolution"
-            ),
-        ),
-    ],
-)
 @pytest.mark.parametrize("data_has_nulls", [True, False])
-def test_export_indivisible_chunking(data_has_nulls, exclude_datetime):
+def test_export_indivisible_chunking(data_has_nulls):
     """
     Test ``.get_chunks(n_chunks)`` when internal PyArrow table's is 'indivisibly chunked'.
 
     The setup for the test is a PyArrow table having one of the chunk consisting of a single row,
     meaning that the chunk can't be subdivide.
     """
-    exclude_dtypes = ["category"]
-    if exclude_datetime:
-        exclude_dtypes += ["datetime"]
-    data = get_data_of_all_types(
-        has_nulls=data_has_nulls, exclude_dtypes=exclude_dtypes
-    )
+    data = get_data_of_all_types(has_nulls=data_has_nulls, exclude_dtypes=["category"])
     pd_df = pandas.DataFrame(data)
     pd_chunks = (pd_df.iloc[:1], pd_df.iloc[1:])