Repository: modin-project/modin

Issue:
BUG: Read_parquet raises AttributeError when given a list of files
**Is your feature request related to a problem? Please describe.**
In pandas support read_parquet multiple files at once, but modin.pandas makes error that "AttributeError: 'list' object has no attribute 'startswith'".
So I have to use pd.concat, but It is too slower than pd.read_parquet at once.


Discussion:
@JoonHong-Kim thank you for reporting this issue. I can reproduce it at Modin commit d724802391524ad21847722d3d92f8448233f500 with the following:

```python
import pandas
import modin.pandas as pd

df1 = pd.DataFrame([1], columns=['a'])
df2 = pd.DataFrame([2], columns=['a'])
df1.to_parquet('/tmp/a1.parquet')
df2.to_parquet('/tmp/a2.parquet')
print(pandas.read_parquet(['/tmp/a.parquet', '/tmp/b.parquet']))
print(pd.read_parquet(['/tmp/a.parquet', '/tmp/b.parquet']))
```

I'll see what we can do to fix this.

<details>
<summary>stack trace</summary>

```python-traceback
---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
Input In [8], in <cell line: 1>()
----> 1 print(pd.read_parquet(['/tmp/a.parquet', '/tmp/b.parquet']))

File ~/software_sources/modin/modin/logging/logger_decorator.py:128, in enable_logging.<locals>.decorator.<locals>.run_and_log(*args, **kwargs)
    113 """
    114 Compute function with logging if Modin logging is enabled.
    115
   (...)
    125 Any
    126 """
    127 if LogMode.get() == "disable":
--> 128     return obj(*args, **kwargs)
    130 logger = get_logger()
    131 logger_level = getattr(logger, log_level)

File ~/software_sources/modin/modin/pandas/io.py:313, in read_parquet(path, engine, columns, storage_options, use_nullable_dtypes, **kwargs)
    309 Engine.subscribe(_update_engine)
    310 from modin.core.execution.dispatching.factories.dispatcher import FactoryDispatcher
    312 return DataFrame(
--> 313     query_compiler=FactoryDispatcher.read_parquet(
    314         path=path,
    315         engine=engine,
    316         columns=columns,
    317         storage_options=storage_options,
    318         use_nullable_dtypes=use_nullable_dtypes,
    319         **kwargs,
    320     )
    321 )

File ~/software_sources/modin/modin/core/execution/dispatching/factories/dispatcher.py:180, in FactoryDispatcher.read_parquet(cls, **kwargs)
    177 @classmethod
    178 @_inherit_docstrings(factories.BaseFactory._read_parquet)
    179 def read_parquet(cls, **kwargs):
--> 180     return cls.get_factory()._read_parquet(**kwargs)

File ~/software_sources/modin/modin/core/execution/dispatching/factories/factories.py:207, in BaseFactory._read_parquet(cls, **kwargs)
    199 @classmethod
    200 @doc(
    201     _doc_io_method_template,
   (...)
    205 )
    206 def _read_parquet(cls, **kwargs):
--> 207     return cls.io_cls.read_parquet(**kwargs)

File ~/software_sources/modin/modin/logging/logger_decorator.py:128, in enable_logging.<locals>.decorator.<locals>.run_and_log(*args, **kwargs)
    113 """
    114 Compute function with logging if Modin logging is enabled.
    115
   (...)
    125 Any
    126 """
    127 if LogMode.get() == "disable":
--> 128     return obj(*args, **kwargs)
    130 logger = get_logger()
    131 logger_level = getattr(logger, log_level)

File ~/software_sources/modin/modin/core/io/file_dispatcher.py:154, in FileDispatcher.read(cls, *args, **kwargs)
    131 @classmethod
    132 def read(cls, *args, **kwargs):
    133     """
    134     Read data according passed `args` and `kwargs`.
    135
   (...)
    152     postprocessing work on the resulting query_compiler object.
    153     """
--> 154     query_compiler = cls._read(*args, **kwargs)
    155     # TODO (devin-petersohn): Make this section more general for non-pandas kernel
    156     # implementations.
    157     if StorageFormat.get() == "Pandas":

File ~/software_sources/modin/modin/logging/logger_decorator.py:128, in enable_logging.<locals>.decorator.<locals>.run_and_log(*args, **kwargs)
    113 """
    114 Compute function with logging if Modin logging is enabled.
    115
   (...)
    125 Any
    126 """
    127 if LogMode.get() == "disable":
--> 128     return obj(*args, **kwargs)
    130 logger = get_logger()
    131 logger_level = getattr(logger, log_level)

File ~/software_sources/modin/modin/core/io/column_stores/parquet_dispatcher.py:649, in ParquetDispatcher._read(cls, path, engine, columns, **kwargs)
    640     if len(partitioned_columns):
    641         return cls.single_worker_read(
    642             path,
    643             engine=engine,
   (...)
    646             **kwargs,
    647         )
--> 649 dataset = cls.get_dataset(path, engine, kwargs.get("storage_options") or {})
    650 index_columns = (
    651     dataset.pandas_metadata.get("index_columns", [])
    652     if dataset.pandas_metadata
    653     else []
    654 )
    655 # If we have columns as None, then we default to reading in all the columns

File ~/software_sources/modin/modin/logging/logger_decorator.py:128, in enable_logging.<locals>.decorator.<locals>.run_and_log(*args, **kwargs)
    113 """
    114 Compute function with logging if Modin logging is enabled.
    115
   (...)
    125 Any
    126 """
    127 if LogMode.get() == "disable":
--> 128     return obj(*args, **kwargs)
    130 logger = get_logger()
    131 logger_level = getattr(logger, log_level)

File ~/software_sources/modin/modin/core/io/column_stores/parquet_dispatcher.py:330, in ParquetDispatcher.get_dataset(cls, path, engine, storage_options)
    328 for engine_class in engine_classes:
    329     try:
--> 330         return engine_class(path, storage_options)
    331     except ImportError as err:
    332         error_msgs += "\n - " + str(err)

File ~/software_sources/modin/modin/core/io/column_stores/parquet_dispatcher.py:68, in ColumnStoreDataset.__init__(self, path, storage_options)
     66 self._fs_path = None
     67 self._fs = None
---> 68 self.dataset = self._init_dataset()
     69 self._row_groups_per_file = None
     70 self._files = None

File ~/software_sources/modin/modin/core/io/column_stores/parquet_dispatcher.py:186, in PyArrowDataset._init_dataset(self)
    182 def _init_dataset(self):  # noqa: GL08
    183     from pyarrow.parquet import ParquetDataset
    185     return ParquetDataset(
--> 186         self.fs_path, filesystem=self.fs, use_legacy_dataset=False
    187     )

File ~/software_sources/modin/modin/core/io/column_stores/parquet_dispatcher.py:130, in ColumnStoreDataset.fs_path(self)
    128         self._fs_path = self.path
    129     else:
--> 130         self._fs, self._fs_path = url_to_fs(self.path, **self.storage_options)
    131 return self._fs_path

File ~/opt/anaconda3/envs/modin-dev/lib/python3.10/site-packages/fsspec/core.py:413, in url_to_fs(url, **kwargs)
    411     update_storage_options(options, kwargs)
    412     fs = cls(**options)
--> 413     urlpath = fs._strip_protocol(url)
    414 return fs, urlpath

File ~/opt/anaconda3/envs/modin-dev/lib/python3.10/site-packages/fsspec/implementations/local.py:194, in LocalFileSystem._strip_protocol(cls, path)
    191 @classmethod
    192 def _strip_protocol(cls, path):
    193     path = stringify_path(path)
--> 194     if path.startswith("file://"):
    195         path = path[7:]
    196     return make_path_posix(path).rstrip("/") or cls.root_marker

AttributeError: 'list' object has no attribute 'startswith'
```

</details>
Yup, I can confirm that we currently don't support `list` objects in the modin Parquet reader. We should try to get this in as soon as we can. 

@JoonHong-Kim one work around would be to put the parquet files in a common folder (with nothing else) and then do `pd.read_parquet('/tmp/parquet_files')`. That should work.

Resolving patch:
diff --git a/modin/core/io/column_stores/parquet_dispatcher.py b/modin/core/io/column_stores/parquet_dispatcher.py
--- a/modin/core/io/column_stores/parquet_dispatcher.py
+++ b/modin/core/io/column_stores/parquet_dispatcher.py
@@ -612,6 +612,13 @@ def _read(cls, path, engine, columns, **kwargs):
         ParquetFile API is used. Please refer to the documentation here
         https://arrow.apache.org/docs/python/parquet.html
         """
+        if isinstance(path, list):
+            # TODO(https://github.com/modin-project/modin/issues/5723): read all
+            # files in parallel.
+            compilers: list[cls.query_compiler_cls] = [
+                cls._read(p, engine, columns, **kwargs) for p in path
+            ]
+            return compilers[0].concat(axis=0, other=compilers[1:], ignore_index=True)
         if isinstance(path, str):
             if os.path.isdir(path):
                 path_generator = os.walk(path)


Test patch:
diff --git a/modin/pandas/test/test_io.py b/modin/pandas/test/test_io.py
--- a/modin/pandas/test/test_io.py
+++ b/modin/pandas/test/test_io.py
@@ -1356,6 +1356,16 @@ def test_read_parquet(
                 columns=columns,
             )
 
+    def test_read_parquet_list_of_files_5698(self, engine, make_parquet_file):
+        if engine == "fastparquet" and os.name == "nt":
+            pytest.xfail(reason="https://github.com/pandas-dev/pandas/issues/51720")
+        with ensure_clean(".parquet") as f1, ensure_clean(
+            ".parquet"
+        ) as f2, ensure_clean(".parquet") as f3:
+            for f in [f1, f2, f3]:
+                make_parquet_file(filename=f)
+            eval_io(fn_name="read_parquet", path=[f1, f2, f3], engine=engine)
+
     @pytest.mark.xfail(
         condition="config.getoption('--simulate-cloud').lower() != 'off'",
         reason="The reason of tests fail in `cloud` mode is unknown for now - issue #3264",