Repository: modin-project/modin

Issue:
Make sure IO functions works with path-like filenames



Discussion:

Patch:
diff --git a/modin/core/execution/ray/implementations/pandas_on_ray/io/io.py b/modin/core/execution/ray/implementations/pandas_on_ray/io/io.py
--- a/modin/core/execution/ray/implementations/pandas_on_ray/io/io.py
+++ b/modin/core/execution/ray/implementations/pandas_on_ray/io/io.py
@@ -16,7 +16,7 @@
 import io
 
 import pandas
-from pandas.io.common import get_handle
+from pandas.io.common import get_handle, stringify_path
 
 from modin.core.execution.ray.common import RayWrapper, SignalActor
 from modin.core.execution.ray.generic.io import RayIO
@@ -154,6 +154,7 @@ def to_csv(cls, qc, **kwargs):
         **kwargs : dict
             Parameters for ``pandas.to_csv(**kwargs)``.
         """
+        kwargs["path_or_buf"] = stringify_path(kwargs["path_or_buf"])
         if not cls._to_csv_check_support(kwargs):
             return RayIO.to_csv(qc, **kwargs)
 
diff --git a/modin/core/execution/unidist/implementations/pandas_on_unidist/io/io.py b/modin/core/execution/unidist/implementations/pandas_on_unidist/io/io.py
--- a/modin/core/execution/unidist/implementations/pandas_on_unidist/io/io.py
+++ b/modin/core/execution/unidist/implementations/pandas_on_unidist/io/io.py
@@ -16,6 +16,7 @@
 import io
 
 import pandas
+from pandas.io.common import get_handle, stringify_path
 
 from modin.core.execution.unidist.common import SignalActor, UnidistWrapper
 from modin.core.execution.unidist.generic.io import UnidistIO
@@ -153,6 +154,7 @@ def to_csv(cls, qc, **kwargs):
         **kwargs : dict
             Parameters for ``pandas.to_csv(**kwargs)``.
         """
+        kwargs["path_or_buf"] = stringify_path(kwargs["path_or_buf"])
         if not cls._to_csv_check_support(kwargs):
             return UnidistIO.to_csv(qc, **kwargs)
 
@@ -196,7 +198,7 @@ def func(df, **kw):  # pragma: no cover
             UnidistWrapper.materialize(signals.wait.remote(partition_idx))
 
             # preparing to write data from the buffer to a file
-            with pandas.io.common.get_handle(
+            with get_handle(
                 path_or_buf,
                 # in case when using URL in implicit text mode
                 # pandas try to open `path_or_buf` in binary mode
diff --git a/modin/core/io/column_stores/feather_dispatcher.py b/modin/core/io/column_stores/feather_dispatcher.py
--- a/modin/core/io/column_stores/feather_dispatcher.py
+++ b/modin/core/io/column_stores/feather_dispatcher.py
@@ -13,6 +13,8 @@
 
 """Module houses `FeatherDispatcher` class, that is used for reading `.feather` files."""
 
+from pandas.io.common import stringify_path
+
 from modin.core.io.column_stores.column_store_dispatcher import ColumnStoreDispatcher
 from modin.core.io.file_dispatcher import OpenFile
 from modin.utils import import_optional_dependency
@@ -47,6 +49,7 @@ def _read(cls, path, columns=None, **kwargs):
         PyArrow feather is used. Please refer to the documentation here
         https://arrow.apache.org/docs/python/api.html#feather-format
         """
+        path = stringify_path(path)
         path = cls.get_path(path)
         if columns is None:
             import_optional_dependency(
diff --git a/modin/core/io/column_stores/parquet_dispatcher.py b/modin/core/io/column_stores/parquet_dispatcher.py
--- a/modin/core/io/column_stores/parquet_dispatcher.py
+++ b/modin/core/io/column_stores/parquet_dispatcher.py
@@ -818,6 +818,7 @@ def write(cls, qc, **kwargs):
         **kwargs : dict
             Parameters for `pandas.to_parquet(**kwargs)`.
         """
+        kwargs["path"] = stringify_path(kwargs["path"])
         output_path = kwargs["path"]
         if not isinstance(output_path, str):
             return cls.base_io.to_parquet(qc, **kwargs)
diff --git a/modin/core/io/text/excel_dispatcher.py b/modin/core/io/text/excel_dispatcher.py
--- a/modin/core/io/text/excel_dispatcher.py
+++ b/modin/core/io/text/excel_dispatcher.py
@@ -19,6 +19,7 @@
 from io import BytesIO
 
 import pandas
+from pandas.io.common import stringify_path
 
 from modin.config import NPartitions
 from modin.core.io.text.text_file_dispatcher import TextFileDispatcher
@@ -47,6 +48,7 @@ def _read(cls, io, **kwargs):
         new_query_compiler : BaseQueryCompiler
             Query compiler with imported data for further processing.
         """
+        io = stringify_path(io)
         if (
             kwargs.get("engine", None) is not None
             and kwargs.get("engine") != "openpyxl"
diff --git a/modin/core/io/text/json_dispatcher.py b/modin/core/io/text/json_dispatcher.py
--- a/modin/core/io/text/json_dispatcher.py
+++ b/modin/core/io/text/json_dispatcher.py
@@ -17,6 +17,7 @@
 
 import numpy as np
 import pandas
+from pandas.io.common import stringify_path
 
 from modin.config import NPartitions
 from modin.core.io.file_dispatcher import OpenFile
@@ -43,6 +44,7 @@ def _read(cls, path_or_buf, **kwargs):
         BaseQueryCompiler
             Query compiler with imported data for further processing.
         """
+        path_or_buf = stringify_path(path_or_buf)
         path_or_buf = cls.get_path_or_buffer(path_or_buf)
         if isinstance(path_or_buf, str):
             if not cls.file_exists(path_or_buf):
diff --git a/modin/core/io/text/text_file_dispatcher.py b/modin/core/io/text/text_file_dispatcher.py
--- a/modin/core/io/text/text_file_dispatcher.py
+++ b/modin/core/io/text/text_file_dispatcher.py
@@ -28,6 +28,7 @@
 import pandas
 import pandas._libs.lib as lib
 from pandas.core.dtypes.common import is_list_like
+from pandas.io.common import stringify_path
 
 from modin.config import NPartitions
 from modin.core.io.file_dispatcher import FileDispatcher, OpenFile
@@ -997,6 +998,7 @@ def _read(cls, filepath_or_buffer, **kwargs):
         new_query_compiler : BaseQueryCompiler
             Query compiler with imported data for further processing.
         """
+        filepath_or_buffer = stringify_path(filepath_or_buffer)
         filepath_or_buffer_md = (
             cls.get_path(filepath_or_buffer)
             if isinstance(filepath_or_buffer, str)
diff --git a/modin/experimental/core/io/pickle/pickle_dispatcher.py b/modin/experimental/core/io/pickle/pickle_dispatcher.py
--- a/modin/experimental/core/io/pickle/pickle_dispatcher.py
+++ b/modin/experimental/core/io/pickle/pickle_dispatcher.py
@@ -17,6 +17,7 @@
 import warnings
 
 import pandas
+from pandas.io.common import stringify_path
 
 from modin.config import NPartitions
 from modin.core.io.file_dispatcher import FileDispatcher
@@ -49,6 +50,7 @@ def _read(cls, filepath_or_buffer, **kwargs):
 
         The number of partitions is equal to the number of input files.
         """
+        filepath_or_buffer = stringify_path(filepath_or_buffer)
         if not (isinstance(filepath_or_buffer, str) and "*" in filepath_or_buffer):
             return cls.single_worker_read(
                 filepath_or_buffer,
@@ -113,6 +115,7 @@ def write(cls, qc, **kwargs):
         **kwargs : dict
             Parameters for ``pandas.to_pickle(**kwargs)``.
         """
+        kwargs["filepath_or_buffer"] = stringify_path(kwargs["filepath_or_buffer"])
         if not (
             isinstance(kwargs["filepath_or_buffer"], str)
             and "*" in kwargs["filepath_or_buffer"]
diff --git a/modin/experimental/core/io/text/custom_text_dispatcher.py b/modin/experimental/core/io/text/custom_text_dispatcher.py
--- a/modin/experimental/core/io/text/custom_text_dispatcher.py
+++ b/modin/experimental/core/io/text/custom_text_dispatcher.py
@@ -14,6 +14,7 @@
 """Module houses `ExperimentalCustomTextDispatcher` class, that is used for reading custom text files."""
 
 import pandas
+from pandas.io.common import stringify_path
 
 from modin.config import NPartitions
 from modin.core.io.file_dispatcher import OpenFile
@@ -46,6 +47,7 @@ def _read(cls, filepath_or_buffer, columns, custom_parser, **kwargs):
         BaseQueryCompiler
             Query compiler with imported data for further processing.
         """
+        filepath_or_buffer = stringify_path(filepath_or_buffer)
         filepath_or_buffer_md = (
             cls.get_path(filepath_or_buffer)
             if isinstance(filepath_or_buffer, str)


Test patch:
diff --git a/modin/experimental/pandas/test/test_io_exp.py b/modin/experimental/pandas/test/test_io_exp.py
--- a/modin/experimental/pandas/test/test_io_exp.py
+++ b/modin/experimental/pandas/test/test_io_exp.py
@@ -14,6 +14,7 @@
 import contextlib
 import glob
 import json
+from pathlib import Path
 
 import numpy as np
 import pandas
@@ -243,11 +244,12 @@ def _pandas_read_csv_glob(path, storage_options):
     Engine.get() not in ("Ray", "Unidist", "Dask"),
     reason=f"{Engine.get()} does not have experimental API",
 )
+@pytest.mark.parametrize("pathlike", [False, True])
 @pytest.mark.parametrize("compression", [None, "gzip"])
 @pytest.mark.parametrize(
     "filename", [test_default_to_pickle_filename, "test_to_pickle*.pkl"]
 )
-def test_distributed_pickling(filename, compression):
+def test_distributed_pickling(filename, compression, pathlike):
     data = test_data["int_data"]
     df = pd.DataFrame(data)
 
@@ -255,6 +257,8 @@ def test_distributed_pickling(filename, compression):
     if compression:
         filename = f"{filename}.gz"
 
+    filename = Path(filename) if pathlike else filename
+
     with (
         warns_that_defaulting_to_pandas()
         if filename_param == test_default_to_pickle_filename
@@ -264,7 +268,7 @@ def test_distributed_pickling(filename, compression):
         pickled_df = pd.read_pickle_distributed(filename, compression=compression)
     df_equals(pickled_df, df)
 
-    pickle_files = glob.glob(filename)
+    pickle_files = glob.glob(str(filename))
     teardown_test_files(pickle_files)
 
 
diff --git a/modin/pandas/test/test_io.py b/modin/pandas/test/test_io.py
--- a/modin/pandas/test/test_io.py
+++ b/modin/pandas/test/test_io.py
@@ -616,8 +616,11 @@ def test_read_csv_iteration(self, iterator):
 
             df_equals(modin_df, pd_df)
 
-    def test_read_csv_encoding_976(self):
+    @pytest.mark.parametrize("pathlike", [False, True])
+    def test_read_csv_encoding_976(self, pathlike):
         file_name = "modin/pandas/test/data/issue_976.csv"
+        if pathlike:
+            file_name = Path(file_name)
         names = [str(i) for i in range(11)]
 
         kwargs = {
@@ -2090,12 +2093,14 @@ def test_read_parquet_relative_to_user_home(make_parquet_file):
 
 @pytest.mark.filterwarnings(default_to_pandas_ignore_string)
 class TestJson:
+    @pytest.mark.parametrize("pathlike", [False, True])
     @pytest.mark.parametrize("lines", [False, True])
-    def test_read_json(self, make_json_file, lines):
+    def test_read_json(self, make_json_file, lines, pathlike):
+        unique_filename = make_json_file(lines=lines)
         eval_io(
             fn_name="read_json",
             # read_json kwargs
-            path_or_buf=make_json_file(lines=lines),
+            path_or_buf=Path(unique_filename) if pathlike else unique_filename,
             lines=lines,
         )
 
@@ -2684,7 +2689,8 @@ def test_to_html(self, tmp_path):
 
 @pytest.mark.filterwarnings(default_to_pandas_ignore_string)
 class TestFwf:
-    def test_fwf_file(self, make_fwf_file):
+    @pytest.mark.parametrize("pathlike", [False, True])
+    def test_fwf_file(self, make_fwf_file, pathlike):
         fwf_data = (
             "id8141  360.242940  149.910199 11950.7\n"
             + "id1594  444.953632  166.985655 11788.4\n"
@@ -2695,7 +2701,12 @@ def test_fwf_file(self, make_fwf_file):
         unique_filename = make_fwf_file(fwf_data=fwf_data)
 
         colspecs = [(0, 6), (8, 20), (21, 33), (34, 43)]
-        df = pd.read_fwf(unique_filename, colspecs=colspecs, header=None, index_col=0)
+        df = pd.read_fwf(
+            Path(unique_filename) if pathlike else unique_filename,
+            colspecs=colspecs,
+            header=None,
+            index_col=0,
+        )
         assert isinstance(df, pd.DataFrame)
 
     @pytest.mark.parametrize(