Repository: modin-project/modin (python)

Issue:
allow I/O function overrides
We do this wrapping in general.py: https://github.com/modin-project/modin/blob/33e2377e786c1a60adc84a945d53ccbbd6068c3b/modin/pandas/general.py#L43

But we should do it for I/O methods too.

Pull request:
FEAT-#7492: Allow I/O function accessors.
Allow I/O function accessors to override modin's I/O functions. Also test that we can override all the modin.pandas.general functions, and fix a bug where synonym pairs like `isnull` and `isna` were using the same set of general accessors.

- [x] first commit message and PR title follow format outlined [here](https://modin.readthedocs.io/en/latest/development/contributing.html#commit-message-formatting)
  > **_NOTE:_**  If you edit the PR title to match this format, you need to add another commit (even if it's empty) or amend your last commit for the CI job that checks the PR title to pick up the new PR title.
- [x] passes `flake8 modin/ asv_bench/benchmarks scripts/doc_checker.py`
- [x] passes `black --check modin/ asv_bench/benchmarks scripts/doc_checker.py`
- [x] signed commit with `git commit -s` <!-- you can amend your commit with a signature via `git commit -amend -s` -->
- [x] Resolves #7492
- [x] tests added and passing
- [x] module layout described at `docs/development/architecture.rst` is up-to-date <!-- if you have added, renamed or removed files or directories please update the documentation accordingly -->


Patch:
diff --git a/modin/pandas/general.py b/modin/pandas/general.py
index 7ea943b1..79f3a027 100644
--- a/modin/pandas/general.py
+++ b/modin/pandas/general.py
@@ -38,10 +38,8 @@ from .dataframe import DataFrame
 from .series import Series
 
 
-@_inherit_docstrings(pandas.isna, apilink="pandas.isna")
 @enable_logging
-@wrap_free_function_in_argument_caster("isna")
-def isna(
+def _isna(
     obj,
 ) -> bool | npt.NDArray[np.bool_] | Series | DataFrame:  # noqa: PR01, RT01, D200
     """
@@ -53,13 +51,15 @@ def isna(
         return pandas.isna(obj)
 
 
-isnull = isna
+_inherit_isna_docstring = _inherit_docstrings(pandas.isnull, apilink="pandas.isna")
+
+isna = _inherit_isna_docstring(wrap_free_function_in_argument_caster("isna")(_isna))
+
+isnull = _inherit_isna_docstring(wrap_free_function_in_argument_caster("isnull")(_isna))
 
 
-@_inherit_docstrings(pandas.notna, apilink="pandas.notna")
 @enable_logging
-@wrap_free_function_in_argument_caster("notna")
-def notna(
+def _notna(
     obj,
 ) -> bool | npt.NDArray[np.bool_] | Series | DataFrame:  # noqa: PR01, RT01, D200
     """
@@ -71,7 +71,13 @@ def notna(
         return pandas.notna(obj)
 
 
-notnull = notna
+_inherit_notna_docstring = _inherit_docstrings(pandas.notna, apilink="pandas.notna")
+
+notnull = _inherit_notna_docstring(
+    wrap_free_function_in_argument_caster("notnull")(_notna)
+)
+
+notna = _inherit_notna_docstring(wrap_free_function_in_argument_caster("notna")(_notna))
 
 
 @_inherit_docstrings(pandas.merge, apilink="pandas.merge")
@@ -768,8 +774,8 @@ def lreshape(data: DataFrame, groups, dropna=True) -> DataFrame:
 
 
 @_inherit_docstrings(pandas.wide_to_long, apilink="pandas.wide_to_long")
-@wrap_free_function_in_argument_caster("wide_to_long")
 @enable_logging
+@wrap_free_function_in_argument_caster("wide_to_long")
 def wide_to_long(
     df: DataFrame, stubnames, i, j, sep: str = "", suffix: str = r"\d+"
 ) -> DataFrame:  # noqa: PR01, RT01, D200
@@ -823,9 +829,9 @@ def _determine_name(objs: Iterable[BaseQueryCompiler], axis: Union[int, str]):
         return None
 
 
-@wrap_free_function_in_argument_caster("to_timedelta")
 @_inherit_docstrings(pandas.to_datetime, apilink="pandas.to_timedelta")
 @enable_logging
+@wrap_free_function_in_argument_caster("to_timedelta")
 def to_timedelta(
     arg, unit=None, errors="raise"
 ) -> Scalar | pandas.Index | Series:  # noqa: PR01, RT01, D200
diff --git a/modin/pandas/io.py b/modin/pandas/io.py
index c43ee436..c6d14c8a 100644
--- a/modin/pandas/io.py
+++ b/modin/pandas/io.py
@@ -68,6 +68,9 @@ from modin.config import ModinNumpy
 from modin.core.dataframe.base.interchange.dataframe_protocol.dataframe import (
     ProtocolDataframe,
 )
+from modin.core.storage_formats.pandas.query_compiler_caster import (
+    wrap_free_function_in_argument_caster,
+)
 from modin.error_message import ErrorMessage
 from modin.logging import ClassLogger, enable_logging
 from modin.utils import (
@@ -131,8 +134,9 @@ def _read(**kwargs):
 
 
 @_inherit_docstrings(pandas.read_xml, apilink="pandas.read_xml")
-@expanduser_path_arg("path_or_buffer")
 @enable_logging
+@wrap_free_function_in_argument_caster("read_xml")
+@expanduser_path_arg("path_or_buffer")
 def read_xml(
     path_or_buffer: FilePath | ReadBuffer[bytes] | ReadBuffer[str],
     *,
@@ -158,8 +162,9 @@ def read_xml(
 
 
 @_inherit_docstrings(pandas.read_csv, apilink="pandas.read_csv")
-@expanduser_path_arg("filepath_or_buffer")
 @enable_logging
+@wrap_free_function_in_argument_caster("read_csv")
+@expanduser_path_arg("filepath_or_buffer")
 def read_csv(
     filepath_or_buffer: FilePath | ReadCsvBuffer[bytes] | ReadCsvBuffer[str],
     *,
@@ -230,8 +235,9 @@ def read_csv(
 
 
 @_inherit_docstrings(pandas.read_table, apilink="pandas.read_table")
-@expanduser_path_arg("filepath_or_buffer")
 @enable_logging
+@wrap_free_function_in_argument_caster("read_table")
+@expanduser_path_arg("filepath_or_buffer")
 def read_table(
     filepath_or_buffer: FilePath | ReadCsvBuffer[bytes] | ReadCsvBuffer[str],
     *,
@@ -304,8 +310,9 @@ def read_table(
 
 
 @_inherit_docstrings(pandas.read_parquet, apilink="pandas.read_parquet")
-@expanduser_path_arg("path")
 @enable_logging
+@wrap_free_function_in_argument_caster("read_parquet")
+@expanduser_path_arg("path")
 def read_parquet(
     path,
     engine: str = "auto",
@@ -340,8 +347,9 @@ def read_parquet(
 
 
 @_inherit_docstrings(pandas.read_json, apilink="pandas.read_json")
-@expanduser_path_arg("path_or_buf")
 @enable_logging
+@wrap_free_function_in_argument_caster("read_json")
+@expanduser_path_arg("path_or_buf")
 def read_json(
     path_or_buf,
     *,
@@ -372,6 +380,7 @@ def read_json(
 
 @_inherit_docstrings(pandas.read_gbq, apilink="pandas.read_gbq")
 @enable_logging
+@wrap_free_function_in_argument_caster("read_gbq")
 def read_gbq(
     query: str,
     project_id: str | None = None,
@@ -396,8 +405,9 @@ def read_gbq(
 
 
 @_inherit_docstrings(pandas.read_html, apilink="pandas.read_html")
-@expanduser_path_arg("io")
 @enable_logging
+@wrap_free_function_in_argument_caster("read_html")
+@expanduser_path_arg("io")
 def read_html(
     io,
     *,
@@ -432,6 +442,7 @@ def read_html(
 
 @_inherit_docstrings(pandas.read_clipboard, apilink="pandas.read_clipboard")
 @enable_logging
+@wrap_free_function_in_argument_caster("read_clipboard")
 def read_clipboard(
     sep=r"\s+",
     dtype_backend: Union[DtypeBackend, NoDefault] = no_default,
@@ -451,8 +462,9 @@ def read_clipboard(
 
 
 @_inherit_docstrings(pandas.read_excel, apilink="pandas.read_excel")
-@expanduser_path_arg("io")
 @enable_logging
+@wrap_free_function_in_argument_caster("read_excel")
+@expanduser_path_arg("io")
 def read_excel(
     io,
     sheet_name: str | int | list[IntStrT] | None = 0,
@@ -500,8 +512,9 @@ def read_excel(
 
 
 @_inherit_docstrings(pandas.read_hdf, apilink="pandas.read_hdf")
-@expanduser_path_arg("path_or_buf")
 @enable_logging
+@wrap_free_function_in_argument_caster("read_hdf")
+@expanduser_path_arg("path_or_buf")
 def read_hdf(
     path_or_buf,
     key=None,
@@ -527,8 +540,9 @@ def read_hdf(
 
 
 @_inherit_docstrings(pandas.read_feather, apilink="pandas.read_feather")
-@expanduser_path_arg("path")
 @enable_logging
+@wrap_free_function_in_argument_caster("read_feather")
+@expanduser_path_arg("path")
 def read_feather(
     path,
     columns: Sequence[Hashable] | None = None,
@@ -546,8 +560,9 @@ def read_feather(
 
 
 @_inherit_docstrings(pandas.read_stata)
-@expanduser_path_arg("filepath_or_buffer")
 @enable_logging
+@wrap_free_function_in_argument_caster("read_stata")
+@expanduser_path_arg("filepath_or_buffer")
 def read_stata(
     filepath_or_buffer,
     *,
@@ -571,8 +586,9 @@ def read_stata(
 
 
 @_inherit_docstrings(pandas.read_sas, apilink="pandas.read_sas")
-@expanduser_path_arg("filepath_or_buffer")
 @enable_logging
+@wrap_free_function_in_argument_caster("read_sas")
+@expanduser_path_arg("filepath_or_buffer")
 def read_sas(
     filepath_or_buffer,
     *,
@@ -602,8 +618,9 @@ def read_sas(
 
 
 @_inherit_docstrings(pandas.read_pickle, apilink="pandas.read_pickle")
-@expanduser_path_arg("filepath_or_buffer")
 @enable_logging
+@wrap_free_function_in_argument_caster("read_pickle")
+@expanduser_path_arg("filepath_or_buffer")
 def read_pickle(
     filepath_or_buffer,
     compression: CompressionOptions = "infer",
@@ -620,6 +637,7 @@ def read_pickle(
 
 @_inherit_docstrings(pandas.read_sql, apilink="pandas.read_sql")
 @enable_logging
+@wrap_free_function_in_argument_caster("read_sql")
 def read_sql(
     sql,
     con,
@@ -650,8 +668,9 @@ def read_sql(
 
 
 @_inherit_docstrings(pandas.read_fwf, apilink="pandas.read_fwf")
-@expanduser_path_arg("filepath_or_buffer")
 @enable_logging
+@wrap_free_function_in_argument_caster("read_fwf")
+@expanduser_path_arg("filepath_or_buffer")
 def read_fwf(
     filepath_or_buffer: Union[str, pathlib.Path, IO[AnyStr]],
     *,
@@ -687,6 +706,7 @@ def read_fwf(
 
 @_inherit_docstrings(pandas.read_sql_table, apilink="pandas.read_sql_table")
 @enable_logging
+@wrap_free_function_in_argument_caster("read_sql_table")
 def read_sql_table(
     table_name,
     con,
@@ -712,6 +732,7 @@ def read_sql_table(
 
 @_inherit_docstrings(pandas.read_sql_query, apilink="pandas.read_sql_query")
 @enable_logging
+@wrap_free_function_in_argument_caster("read_sql_query")
 def read_sql_query(
     sql,
     con,
@@ -733,8 +754,9 @@ def read_sql_query(
 
 
 @_inherit_docstrings(pandas.to_pickle)
-@expanduser_path_arg("filepath_or_buffer")
 @enable_logging
+@wrap_free_function_in_argument_caster("to_pickle")
+@expanduser_path_arg("filepath_or_buffer")
 def to_pickle(
     obj: Any,
     filepath_or_buffer,
@@ -756,8 +778,9 @@ def to_pickle(
 
 
 @_inherit_docstrings(pandas.read_spss, apilink="pandas.read_spss")
-@expanduser_path_arg("path")
 @enable_logging
+@wrap_free_function_in_argument_caster("read_spss")
+@expanduser_path_arg("path")
 def read_spss(
     path: Union[str, pathlib.Path],
     usecols: Optional[Sequence[str]] = None,
@@ -781,6 +804,7 @@ def read_spss(
 
 @_inherit_docstrings(pandas.json_normalize, apilink="pandas.json_normalize")
 @enable_logging
+@wrap_free_function_in_argument_caster("json_normalize")
 def json_normalize(
     data: Union[Dict, List[Dict]],
     record_path: Optional[Union[str, List]] = None,
@@ -803,8 +827,9 @@ def json_normalize(
 
 
 @_inherit_docstrings(pandas.read_orc, apilink="pandas.read_orc")
-@expanduser_path_arg("path")
 @enable_logging
+@wrap_free_function_in_argument_caster("read_orc")
+@expanduser_path_arg("path")
 def read_orc(
     path,
     columns: Optional[List[str]] = None,
@@ -950,6 +975,7 @@ class ExcelFile(ClassLogger, pandas.ExcelFile):  # noqa: PR01, D200
         return method
 
 
+@wrap_free_function_in_argument_caster("from_non_pandas")
 def from_non_pandas(df, index, columns, dtype) -> DataFrame | None:
     """
     Convert a non-pandas DataFrame into Modin DataFrame.
@@ -978,6 +1004,7 @@ def from_non_pandas(df, index, columns, dtype) -> DataFrame | None:
     return new_qc
 
 
+@wrap_free_function_in_argument_caster("from_pandas")
 def from_pandas(df) -> DataFrame:
     """
     Convert a pandas DataFrame to a Modin DataFrame.
@@ -997,6 +1024,7 @@ def from_pandas(df) -> DataFrame:
     return ModinObjects.DataFrame(query_compiler=FactoryDispatcher.from_pandas(df))
 
 
+@wrap_free_function_in_argument_caster("from_arrow")
 def from_arrow(at) -> DataFrame:
     """
     Convert an Arrow Table to a Modin DataFrame.
@@ -1016,6 +1044,7 @@ def from_arrow(at) -> DataFrame:
     return ModinObjects.DataFrame(query_compiler=FactoryDispatcher.from_arrow(at))
 
 
+@wrap_free_function_in_argument_caster("from_dataframe")
 def from_dataframe(df: ProtocolDataframe) -> DataFrame:
     """
     Convert a DataFrame implementing the dataframe interchange protocol to a Modin DataFrame.
@@ -1039,6 +1068,7 @@ def from_dataframe(df: ProtocolDataframe) -> DataFrame:
     )
 
 
+@wrap_free_function_in_argument_caster("from_ray")
 def from_ray(ray_obj) -> DataFrame:
     """
     Convert a Ray Dataset into Modin DataFrame.
@@ -1062,6 +1092,7 @@ def from_ray(ray_obj) -> DataFrame:
     return ModinObjects.DataFrame(query_compiler=FactoryDispatcher.from_ray(ray_obj))
 
 
+@wrap_free_function_in_argument_caster("from_dask")
 def from_dask(dask_obj) -> DataFrame:
     """
     Convert a Dask DataFrame to a Modin DataFrame.
@@ -1085,6 +1116,7 @@ def from_dask(dask_obj) -> DataFrame:
     return ModinObjects.DataFrame(query_compiler=FactoryDispatcher.from_dask(dask_obj))
 
 
+@wrap_free_function_in_argument_caster("from_map")
 def from_map(func, iterable, *args, **kwargs) -> DataFrame:
     """
     Create a Modin DataFrame from map function applied to an iterable object.
@@ -1115,6 +1147,7 @@ def from_map(func, iterable, *args, **kwargs) -> DataFrame:
     )
 
 
+@wrap_free_function_in_argument_caster("to_pandas")
 def to_pandas(modin_obj: SupportsPublicToPandas) -> DataFrame | Series:
     """
     Convert a Modin DataFrame/Series to a pandas DataFrame/Series.
@@ -1132,6 +1165,7 @@ def to_pandas(modin_obj: SupportsPublicToPandas) -> DataFrame | Series:
     return modin_obj._to_pandas()
 
 
+@wrap_free_function_in_argument_caster("to_numpy")
 def to_numpy(
     modin_obj: Union[SupportsPrivateToNumPy, SupportsPublicToNumPy],
 ) -> np.ndarray:
@@ -1156,6 +1190,7 @@ def to_numpy(
     return array
 
 
+@wrap_free_function_in_argument_caster("to_ray")
 def to_ray(modin_obj):
     """
     Convert a Modin DataFrame/Series to a Ray Dataset.
@@ -1179,6 +1214,7 @@ def to_ray(modin_obj):
     return FactoryDispatcher.to_ray(modin_obj)
 
 
+@wrap_free_function_in_argument_caster("to_dask")
 def to_dask(modin_obj):
     """
     Convert a Modin DataFrame/Series to a Dask DataFrame/Series.


Test patch:
diff --git a/modin/tests/pandas/extensions/test_pd_extensions.py b/modin/tests/pandas/extensions/test_pd_extensions.py
index 10d1b8f5..2d8f0a7c 100644
--- a/modin/tests/pandas/extensions/test_pd_extensions.py
+++ b/modin/tests/pandas/extensions/test_pd_extensions.py
@@ -12,6 +12,7 @@
 # governing permissions and limitations under the License.
 
 import re
+from types import FunctionType
 
 import pandas
 import pytest
@@ -22,6 +23,28 @@ from modin.pandas.api.extensions import register_pd_accessor
 from modin.tests.pandas.utils import df_equals, eval_general
 
 
+@pytest.fixture(
+    params=sorted(
+        key
+        for key, value in pd.__dict__.items()
+        if isinstance(value, FunctionType) and value.__module__ == pd.general.__name__
+    )
+)
+def pd_general_function(request):
+    return request.param
+
+
+@pytest.fixture(
+    params=sorted(
+        key
+        for key, value in pd.__dict__.items()
+        if isinstance(value, FunctionType) and value.__module__ == pd.io.__name__
+    )
+)
+def pd_io_function(request):
+    return request.param
+
+
 class TestRegisterForAllBackends:
     def test_add_new_function(self):
         expected_string_val = "Some string value"
@@ -39,6 +62,16 @@ class TestRegisterForAllBackends:
         register_pd_accessor(attribute_name)(expected_val)
         assert pd.four == expected_val
 
+    def test_override_io_function(self, pd_io_function):
+        sentinel = object()
+        register_pd_accessor(pd_io_function)(lambda: sentinel)
+        assert getattr(pd, pd_io_function)() == sentinel
+
+    def test_override_general_function(self, pd_general_function):
+        sentinel = object()
+        register_pd_accessor(pd_general_function)(lambda: sentinel)
+        assert getattr(pd, pd_general_function)() == sentinel
+
 
 class TestRegisterForOneBackend:
     def test_add_new_function(self):