Repository: modin-project/modin

Issue:
ValueError: The 'nrows' option is not supported with the 'pyarrow' engine
The pandas.read_csv method supports ‘c’, ‘python’, ‘pyarrow’ engines. It looks like modin is not supported when using 'pyarrow' engine.

Intel(R) Extension for Scikit-learn* enabled (https://github.com/intel/scikit-learn-intelex)
2024-06-18 05:01:34,908 INFO worker.py:1753 -- Started a local Ray instance.
Traceback (most recent call last):
  File "/home/ad/anomaly_detection.py", line 41, in <module>
    model_fitting()
  File "/home/ad/anomaly_detection.py", line 37, in model_fitting
    raise e
  File "/home/ad/anomaly_detection.py", line 14, in model_fitting
    data_csv = pd.read_csv('./data.csv', engine='pyarrow')
  File "/home/ad/analytics_env/lib/python3.10/site-packages/modin/utils.py", line 511, in wrapped
    return func(*params.args, **params.kwargs)
  File "/home/ad/analytics_env/lib/python3.10/site-packages/modin/logging/logger_decorator.py", line 125, in run_and_log
    return obj(*args, **kwargs)
  File "/home/ad/analytics_env/lib/python3.10/site-packages/modin/pandas/io.py", line 227, in read_csv
    return _read(**kwargs)
  File "/home/ad/analytics_env/lib/python3.10/site-packages/modin/pandas/io.py", line 117, in _read
    pd_obj = FactoryDispatcher.read_csv(**kwargs)
  File "/home/ad/analytics_env/lib/python3.10/site-packages/modin/core/execution/dispatching/factories/dispatcher.py", line 207, in read_csv
    return cls.get_factory()._read_csv(**kwargs)
  File "/home/ad/analytics_env/lib/python3.10/site-packages/modin/core/execution/dispatching/factories/factories.py", line 268, in _read_csv
    return cls.io_cls.read_csv(**kwargs)
  File "/home/ad/analytics_env/lib/python3.10/site-packages/modin/logging/logger_decorator.py", line 125, in run_and_log
    return obj(*args, **kwargs)
  File "/home/ad/analytics_env/lib/python3.10/site-packages/modin/core/io/file_dispatcher.py", line 159, in read
    query_compiler = cls._read(*args, **kwargs)
  File "/home/ad/analytics_env/lib/python3.10/site-packages/modin/logging/logger_decorator.py", line 125, in run_and_log
    return obj(*args, **kwargs)
  File "/home/ad/analytics_env/lib/python3.10/site-packages/modin/core/io/text/text_file_dispatcher.py", line 1068, in _read
    pd_df_metadata = cls.read_callback(
  File "/home/ad/analytics_env/lib/python3.10/site-packages/modin/logging/logger_decorator.py", line 125, in run_and_log
    return obj(*args, **kwargs)
  File "/home/ad/analytics_env/lib/python3.10/site-packages/modin/core/storage_formats/pandas/parsers.py", line 381, in read_callback
    return pandas.read_csv(*args, **kwargs)
  File "/home/ad/analytics_env/lib/python3.10/site-packages/pandas/io/parsers/readers.py", line 1026, in read_csv
    return _read(filepath_or_buffer, kwds)
  File "/home/ad/analytics_env/lib/python3.10/site-packages/pandas/io/parsers/readers.py", line 620, in _read
    parser = TextFileReader(filepath_or_buffer, **kwds)
  File "/home/ad/analytics_env/lib/python3.10/site-packages/pandas/io/parsers/readers.py", line 1607, in __init__
    options = self._get_options_with_defaults(engine)
  File "/home/ad/analytics_env/lib/python3.10/site-packages/pandas/io/parsers/readers.py", line 1643, in _get_options_with_defaults
    raise ValueError(
ValueError: The 'nrows' option is not supported with the 'pyarrow' engine



Discussion:
@anmyachev, could you take a look at this?
Hi @azhuvath! Could you provide modin, pandas and pyarrow versions?
Current situation with unsupported `read_csv` parameters with pyarrow: https://github.com/pandas-dev/pandas/issues/38872. Upstream pandas does not support `nrows` parameter.
I found an issue in Modin.

Resolving patch:
diff --git a/modin/core/io/text/text_file_dispatcher.py b/modin/core/io/text/text_file_dispatcher.py
--- a/modin/core/io/text/text_file_dispatcher.py
+++ b/modin/core/io/text/text_file_dispatcher.py
@@ -216,7 +216,7 @@ def partitioned_file(
         newline: bytes = None,
         header_size: int = 0,
         pre_reading: int = 0,
-        read_callback_kw: dict = None,
+        get_metadata_kw: dict = None,
     ):
         """
         Compute chunk sizes in bytes for every partition.
@@ -244,7 +244,7 @@ def partitioned_file(
             Number of rows, that occupied by header.
         pre_reading : int, default: 0
             Number of rows between header and skipped rows, that should be read.
-        read_callback_kw : dict, optional
+        get_metadata_kw : dict, optional
             Keyword arguments for `cls.read_callback` to compute metadata if needed.
             This option is not compatible with `pre_reading!=0`.
 
@@ -255,11 +255,11 @@ def partitioned_file(
                 int : partition start read byte
                 int : partition end read byte
         pandas.DataFrame or None
-            Dataframe from which metadata can be retrieved. Can be None if `read_callback_kw=None`.
+            Dataframe from which metadata can be retrieved. Can be None if `get_metadata_kw=None`.
         """
-        if read_callback_kw is not None and pre_reading != 0:
+        if get_metadata_kw is not None and pre_reading != 0:
             raise ValueError(
-                f"Incompatible combination of parameters: {read_callback_kw=}, {pre_reading=}"
+                f"Incompatible combination of parameters: {get_metadata_kw=}, {pre_reading=}"
             )
         read_rows_counter = 0
         outside_quotes = True
@@ -297,11 +297,11 @@ def partitioned_file(
             rows_skipper(skiprows)
         else:
             rows_skipper(skiprows)
-            if read_callback_kw:
+            if get_metadata_kw:
                 start = f.tell()
                 # For correct behavior, if we want to avoid double skipping rows,
                 # we need to get metadata after skipping.
-                pd_df_metadata = cls.read_callback(f, **read_callback_kw)
+                pd_df_metadata = cls.read_callback(f, **get_metadata_kw)
                 f.seek(start)
             rows_skipper(header_size)
 
@@ -1063,28 +1063,32 @@ def _read(cls, filepath_or_buffer, **kwargs):
             and (usecols is None or skiprows is None)
             and pre_reading == 0
         )
-        read_callback_kw = dict(kwargs, nrows=1, skipfooter=0, index_col=index_col)
+        get_metadata_kw = dict(kwargs, nrows=1, skipfooter=0, index_col=index_col)
+        if get_metadata_kw.get("engine", None) == "pyarrow":
+            # pyarrow engine doesn't support `nrows` option;
+            # https://github.com/pandas-dev/pandas/issues/38872 can be used to track pyarrow engine features
+            get_metadata_kw["engine"] = "c"
         if not can_compute_metadata_while_skipping_rows:
             pd_df_metadata = cls.read_callback(
                 filepath_or_buffer_md,
-                **read_callback_kw,
+                **get_metadata_kw,
             )
             column_names = pd_df_metadata.columns
             column_widths, num_splits = cls._define_metadata(
                 pd_df_metadata, column_names
             )
-            read_callback_kw = None
+            get_metadata_kw = None
         else:
-            read_callback_kw = dict(read_callback_kw, skiprows=None)
+            get_metadata_kw = dict(get_metadata_kw, skiprows=None)
             # `memory_map` doesn't work with file-like object so we can't use it here.
             # We can definitely skip it without violating the reading logic
             # since this parameter is intended to optimize reading.
             # For reading a couple of lines, this is not essential.
-            read_callback_kw.pop("memory_map", None)
+            get_metadata_kw.pop("memory_map", None)
             # These parameters are already used when opening file `f`,
             # they do not need to be used again.
-            read_callback_kw.pop("storage_options", None)
-            read_callback_kw.pop("compression", None)
+            get_metadata_kw.pop("storage_options", None)
+            get_metadata_kw.pop("compression", None)
 
         with OpenFile(
             filepath_or_buffer_md,
@@ -1110,7 +1114,7 @@ def _read(cls, filepath_or_buffer, **kwargs):
                 newline=newline,
                 header_size=header_size,
                 pre_reading=pre_reading,
-                read_callback_kw=read_callback_kw,
+                get_metadata_kw=get_metadata_kw,
             )
             if can_compute_metadata_while_skipping_rows:
                 pd_df_metadata = pd_df_metadata_temp


Test patch:
diff --git a/modin/tests/pandas/test_io.py b/modin/tests/pandas/test_io.py
--- a/modin/tests/pandas/test_io.py
+++ b/modin/tests/pandas/test_io.py
@@ -653,7 +653,7 @@ def test_read_csv_encoding_976(self, pathlike):
     # Quoting, Compression parameters tests
     @pytest.mark.parametrize("compression", ["infer", "gzip", "bz2", "xz", "zip"])
     @pytest.mark.parametrize("encoding", [None, "latin8", "utf16"])
-    @pytest.mark.parametrize("engine", [None, "python", "c"])
+    @pytest.mark.parametrize("engine", [None, "python", "c", "pyarrow"])
     def test_read_csv_compression(self, make_csv_file, compression, encoding, engine):
         unique_filename = make_csv_file(encoding=encoding, compression=compression)
         expected_exception = None